diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/code_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..f320ad8d08bb3fdb86f4ea6da8ca434213eff940 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 127.60186833333333, + "recon_r2": 0.9549164696200511, + "l1_per_token": 607.733303906817 + }, + "seed1": { + "l0": 128.26901333333333, + "recon_r2": 0.9550364366052586, + "l1_per_token": 610.3533042558479 + }, + "seed2": { + "l0": 127.33541666666666, + "recon_r2": 0.9548217346630481, + "l1_per_token": 606.1776392972702 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7197823649387445, + "mcc_abs": 0.7198161701249538, + "act_iou": 0.3765977592859709, + "act_iou_n_pairs": 12288, + "input_r2": 0.9726636356267777, + "matched_mcc_signed": { + "0.6": 0.8604219640832733, + "0.7": 0.8902659821617671, + "0.8": 0.9180785324351199, + "0.9": 0.9493570893912988 + }, + "matched_mcc_abs": { + "0.6": 0.8604668676010798, + "0.7": 0.8905839594546318, + "0.8": 0.9183884360757144, + "0.9": 0.9498763732181995 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7164556125489968, + "mcc_abs": 0.7164774781849763, + "act_iou": 0.37359442046385594, + "act_iou_n_pairs": 12288, + "input_r2": 0.9724552338792009, + "matched_mcc_signed": { + "0.6": 0.8593533231762739, + "0.7": 0.8896758148365826, + "0.8": 0.9198472560480646, + "0.9": 0.9504334646407947 + }, + "matched_mcc_abs": { + "0.6": 0.8599169115633304, + "0.7": 0.890333888426092, + "0.8": 0.9201968236292594, + "0.9": 0.9505123240419678 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7184330381742603, + "mcc_abs": 0.7184667161459387, + "act_iou": 0.3755397594742718, + "act_iou_n_pairs": 12288, + "input_r2": 0.9727663797231461, + "matched_mcc_signed": { + "0.6": 0.8577225733053204, + "0.7": 0.8891815496720875, + "0.8": 0.9174585580331756, + "0.9": 0.9486653999848231 + }, + "matched_mcc_abs": { + "0.6": 0.8577840632335128, + "0.7": 0.8894980949335266, + "0.8": 0.9177318409381179, + "0.9": 0.9487187715656917 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 127.73543277777777, + "l0_std": 0.48091608375848277, + "recon_r2_mean": 0.9549248802961192, + "recon_r2_std": 0.00010759779552195764, + "l1_per_token_mean": 608.088082486645, + "l1_per_token_std": 2.1103187773656535, + "input_r2_mean": 0.9726284164097082, + "input_r2_std": 0.00015853463025262014, + "mcc_signed_mean": 0.7182236718873338, + "mcc_signed_std": 0.00167322922734722, + "mcc_abs_mean": 0.718253454818623, + "mcc_abs_std": 0.0016795315605293117, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8591659535216225, + "0.7": 0.889707782223479, + "0.8": 0.9184614488387867, + "0.9": 0.9494853180056388 + }, + "matched_mcc_signed_band": 0.9042051256473818, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8593892807993077, + "0.7": 0.8901386476047501, + "0.8": 0.9187723668810305, + "0.9": 0.9497024896086197 + }, + "matched_mcc_abs_band": 0.904500696223427, + "act_iou_mean": 0.3752439797413662, + "act_iou_std": 0.0015233598583079414, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/dict_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..a51dfb1c87e326bb4f48a6801fa78fcd97e6d5d9 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 128, + "l0_raw": 127.7182478515625, + "recon_r2": 0.9548467507110822, + "l1_per_token": 607.7213934281276, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 242, + "dense_rate": 0.019694010416666668, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 470, + "0.6": 95, + "0.7": 21, + "0.8": 5, + "0.9": 2 + }, + "n_dup_clusters_signed": { + "0.5": 232, + "0.6": 47, + "0.7": 10, + "0.8": 2, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 2 + }, + "babel_mu1_signed": 0.3079283535480499, + "n_dup_abs": { + "0.5": 1342, + "0.6": 915, + "0.7": 807, + "0.8": 751, + "0.9": 656 + }, + "n_dup_clusters_abs": { + "0.5": 667, + "0.6": 456, + "0.7": 403, + "0.8": 375, + "0.9": 328 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 2 + }, + "babel_mu1_abs": 0.36509886384010315 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 128, + "l0_raw": 128.3524513671875, + "recon_r2": 0.9549618942987953, + "l1_per_token": 610.2288667752479, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 238, + "dense_rate": 0.019368489583333332, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 497, + "0.6": 95, + "0.7": 19, + "0.8": 5, + "0.9": 3 + }, + "n_dup_clusters_signed": { + "0.5": 245, + "0.6": 46, + "0.7": 9, + "0.8": 2, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 3 + }, + "babel_mu1_signed": 0.30881133675575256, + "n_dup_abs": { + "0.5": 1375, + "0.6": 928, + "0.7": 807, + "0.8": 739, + "0.9": 631 + }, + "n_dup_clusters_abs": { + "0.5": 681, + "0.6": 461, + "0.7": 402, + "0.8": 369, + "0.9": 315 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 3 + }, + "babel_mu1_abs": 0.3659749925136566 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 127, + "l0_raw": 127.48565991210937, + "recon_r2": 0.954758292979353, + "l1_per_token": 606.2793811777946, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 238, + "dense_rate": 0.019368489583333332, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 532, + "0.6": 123, + "0.7": 31, + "0.8": 9, + "0.9": 2 + }, + "n_dup_clusters_signed": { + "0.5": 261, + "0.6": 61, + "0.7": 15, + "0.8": 4, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 4, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 2 + }, + "babel_mu1_signed": 0.30918964743614197, + "n_dup_abs": { + "0.5": 1405, + "0.6": 943, + "0.7": 785, + "0.8": 713, + "0.9": 638 + }, + "n_dup_clusters_abs": { + "0.5": 694, + "0.6": 469, + "0.7": 390, + "0.8": 356, + "0.9": 319 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 2 + }, + "babel_mu1_abs": 0.36544737219810486 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.7136623859405518, + "dcs_abs": 0.7151963114738464, + "frac_sign_flipped": 0.022786458333333332, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7085920572280884, + "0.6": 0.7129131555557251, + "0.7": 0.7134965658187866, + "0.8": 0.7136114835739136, + "0.9": 0.7136274576187134 + }, + "unique_frac_signed": { + "0.5": 0.9800618489583334, + "0.6": 0.9960530598958334, + "0.7": 0.9991455078125, + "0.8": 0.999755859375, + "0.9": 0.9998779296875 + }, + "dedup_dcs_signed": { + "0.5": 0.6944640417641494, + "0.6": 0.7100993300312742, + "0.7": 0.7128868885774864, + "0.8": 0.7134372620203067, + "0.9": 0.7135403448919533 + }, + "matched_frac_signed": { + "0.5": 0.7732747395833334, + "0.6": 0.7207845052083334, + "0.7": 0.6289876302083334, + "0.8": 0.5049641927083334, + "0.9": 0.2537434895833333 + }, + "matched_cos_abs": { + "0.5": 0.7161705493927002, + "0.6": 0.7201769948005676, + "0.7": 0.7203535437583923, + "0.8": 0.7201673984527588, + "0.9": 0.7195829153060913 + }, + "unique_frac_abs": { + "0.5": 0.9442952473958334, + "0.6": 0.9623209635416666, + "0.7": 0.9670817057291666, + "0.8": 0.9696451822916666, + "0.9": 0.9737955729166666 + }, + "dedup_dcs_abs": { + "0.5": 0.6762764461163897, + "0.6": 0.693041419557024, + "0.7": 0.696640733825916, + "0.8": 0.6983068483532406, + "0.9": 0.7007266572715404 + }, + "matched_frac_abs": { + "0.5": 0.7564290364583334, + "0.6": 0.7080078125, + "0.7": 0.6189778645833334, + "0.8": 0.4969889322916667, + "0.9": 0.24943033854166666 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.7091007828712463, + "dcs_abs": 0.7105391621589661, + "frac_sign_flipped": 0.025797526041666668, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7051715850830078, + "0.6": 0.7086870670318604, + "0.7": 0.7091602683067322, + "0.8": 0.7090668082237244, + "0.9": 0.7090792059898376 + }, + "unique_frac_signed": { + "0.5": 0.9792887369791666, + "0.6": 0.9955240885416666, + "0.7": 0.9989013671875, + "0.8": 0.9996744791666666, + "0.9": 0.9999186197916666 + }, + "dedup_dcs_signed": { + "0.5": 0.6905665909095356, + "0.6": 0.7055150464681598, + "0.7": 0.7083811615666491, + "0.8": 0.7088359922054224, + "0.9": 0.7090215009763293 + }, + "matched_frac_signed": { + "0.5": 0.7677408854166666, + "0.6": 0.7119140625, + "0.7": 0.6239420572916666, + "0.8": 0.4934895833333333, + "0.9": 0.2490234375 + }, + "matched_cos_abs": { + "0.5": 0.7125113606452942, + "0.6": 0.7156597971916199, + "0.7": 0.7157593965530396, + "0.8": 0.7155851125717163, + "0.9": 0.715090811252594 + }, + "unique_frac_abs": { + "0.5": 0.943603515625, + "0.6": 0.9620361328125, + "0.7": 0.9674886067708334, + "0.8": 0.9701741536458334, + "0.9": 0.9736735026041666 + }, + "dedup_dcs_abs": { + "0.5": 0.6723282248276519, + "0.6": 0.688490583699604, + "0.7": 0.6924890613542327, + "0.8": 0.6942421809508232, + "0.9": 0.6962649748723682 + }, + "matched_frac_abs": { + "0.5": 0.750244140625, + "0.6": 0.6988118489583334, + "0.7": 0.614501953125, + "0.8": 0.486572265625, + "0.9": 0.2451171875 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.7128297686576843, + "dcs_abs": 0.7144290804862976, + "frac_sign_flipped": 0.027262369791666668, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7082353830337524, + "0.6": 0.7123907804489136, + "0.7": 0.7128813862800598, + "0.8": 0.7128597497940063, + "0.9": 0.7127940654754639 + }, + "unique_frac_signed": { + "0.5": 0.9787190755208334, + "0.6": 0.9954833984375, + "0.7": 0.9989420572916666, + "0.8": 0.9996744791666666, + "0.9": 0.9998779296875 + }, + "dedup_dcs_signed": { + "0.5": 0.6931634793339375, + "0.6": 0.7091731951368274, + "0.7": 0.7121271986155382, + "0.8": 0.7126276990942035, + "0.9": 0.7127070544811431 + }, + "matched_frac_signed": { + "0.5": 0.7713216145833334, + "0.6": 0.7215983072916666, + "0.7": 0.6263020833333334, + "0.8": 0.503173828125, + "0.9": 0.2534993489583333 + }, + "matched_cos_abs": { + "0.5": 0.7156670689582825, + "0.6": 0.719334065914154, + "0.7": 0.7195640206336975, + "0.8": 0.7192935943603516, + "0.9": 0.7184290885925293 + }, + "unique_frac_abs": { + "0.5": 0.9428304036458334, + "0.6": 0.9617106119791666, + "0.7": 0.9674479166666666, + "0.8": 0.9704182942708334, + "0.9": 0.9741617838541666 + }, + "dedup_dcs_abs": { + "0.5": 0.6747526715019679, + "0.6": 0.6917912047477632, + "0.7": 0.6961407126703609, + "0.8": 0.6980156629191091, + "0.9": 0.6998661625160214 + }, + "matched_frac_abs": { + "0.5": 0.7534993489583334, + "0.6": 0.70947265625, + "0.7": 0.616943359375, + "0.8": 0.4955240885416667, + "0.9": 0.24951171875 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 239, + "n_dense_std": 2.3094010767585034, + "l0_mean": 128, + "l0_std": 0.4486430162394102, + "recon_r2_mean": 0.95485564599641, + "recon_r2_std": null, + "l1_per_token_mean": 608.0765471270566, + "l1_per_token_std": 1.9985449960435793, + "babel_mu1_signed_mean": 0.3086431125799815, + "babel_mu1_signed_std": 0.00064725582115573, + "n_dup_signed_per_threshold": { + "0.5": 499.6666666666667, + "0.6": 104.33333333333333, + "0.7": 23.666666666666668, + "0.8": 6.333333333333333, + "0.9": 2.3333333333333335 + }, + "n_dup_signed_band": 34.16666666666667, + "n_dup_signed_band_std": 6.135620044733321, + "n_dup_clusters_signed_per_threshold": { + "0.5": 246.0, + "0.6": 51.333333333333336, + "0.7": 11.333333333333334, + "0.8": 2.6666666666666665, + "0.9": 1.0 + }, + "n_dup_clusters_signed_band": 16.583333333333336, + "n_dup_clusters_signed_band_std": 3.185252475602732, + "largest_dup_signed_per_threshold": { + "0.5": 3.3333333333333335, + "0.6": 3.0, + "0.7": 3.0, + "0.8": 3.0, + "0.9": 2.3333333333333335 + }, + "largest_dup_signed_band": 2.8333333333333335, + "largest_dup_signed_band_std": 0.14433756729740643, + "dcs_signed_mean": 0.7118643124898275, + "dcs_signed_std": 0.0024292251541414983, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.6927313706692075, + "0.6": 0.7082625238787538, + "0.7": 0.7111317495865578, + "0.8": 0.7116336511066441, + "0.9": 0.7117563001164752 + }, + "dedup_dcs_signed_band": 0.7106960561721076, + "dedup_dcs_signed_band_std": 0.0024241539848505723, + "matched_cos_signed_per_threshold": { + "0.5": 0.7073330084482828, + "0.6": 0.7113303343454996, + "0.7": 0.7118460734685262, + "0.8": 0.7118460138638815, + "0.9": 0.7118335763613383 + }, + "matched_cos_signed_band": 0.7117139995098114, + "matched_cos_signed_band_std": 0.0023763298445997004, + "unique_frac_signed_per_threshold": { + "0.5": 0.9793565538194445, + "0.6": 0.9956868489583334, + "0.7": 0.9989963107638888, + "0.8": 0.9997016059027777, + "0.9": 0.9998914930555555 + }, + "unique_frac_signed_band": 0.9985690646701388, + "unique_frac_signed_band_std": 0.00012050615535432782, + "matched_frac_signed_per_threshold": { + "0.5": 0.7707790798611112, + "0.6": 0.7180989583333334, + "0.7": 0.6264105902777778, + "0.8": 0.5005425347222222, + "0.9": 0.2520887586805555 + }, + "matched_frac_signed_band": 0.5242852105034722, + "matched_frac_signed_band_std": 0.0040934190861738485, + "babel_mu1_abs_mean": 0.3655070761839549, + "babel_mu1_abs_std": 0.00044110518881602777, + "n_dup_abs_per_threshold": { + "0.5": 1374.0, + "0.6": 928.6666666666666, + "0.7": 799.6666666666666, + "0.8": 734.3333333333334, + "0.9": 641.6666666666666 + }, + "n_dup_abs_band": 776.0833333333333, + "n_dup_abs_band_std": 6.251666444503684, + "n_dup_clusters_abs_per_threshold": { + "0.5": 680.6666666666666, + "0.6": 462.0, + "0.7": 398.3333333333333, + "0.8": 366.6666666666667, + "0.9": 320.6666666666667 + }, + "n_dup_clusters_abs_band": 386.9166666666667, + "n_dup_clusters_abs_band_std": 3.50297492616395, + "largest_dup_abs_per_threshold": { + "0.5": 3.3333333333333335, + "0.6": 3.0, + "0.7": 3.0, + "0.8": 3.0, + "0.9": 2.3333333333333335 + }, + "largest_dup_abs_band": 2.8333333333333335, + "largest_dup_abs_band_std": 0.14433756729740643, + "dcs_abs_mean": 0.71338818470637, + "dcs_abs_std": 0.0024969697542773614, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6744524474820032, + "0.6": 0.6911077360014638, + "0.7": 0.6950901692835032, + "0.8": 0.6968548974077243, + "0.9": 0.6989525982199766 + }, + "dedup_dcs_abs_band": 0.695501350228167, + "dedup_dcs_abs_band_std": 0.0023060516767600663, + "matched_cos_abs_per_threshold": { + "0.5": 0.7147829929987589, + "0.6": 0.7183902859687805, + "0.7": 0.7185589869817098, + "0.8": 0.7183487017949423, + "0.9": 0.7177009383837382 + }, + "matched_cos_abs_band": 0.7182497282822927, + "matched_cos_abs_band_std": 0.002404664994481575, + "unique_frac_abs_per_threshold": { + "0.5": 0.943576388888889, + "0.6": 0.9620225694444443, + "0.7": 0.9673394097222222, + "0.8": 0.9700792100694445, + "0.9": 0.973876953125 + }, + "unique_frac_abs_band": 0.9683295355902778, + "unique_frac_abs_band_std": 0.0001125126140885089, + "matched_frac_abs_per_threshold": { + "0.5": 0.753390842013889, + "0.6": 0.7054307725694445, + "0.7": 0.6168077256944445, + "0.8": 0.4930284288194445, + "0.9": 0.24801974826388887 + }, + "matched_frac_abs_band": 0.5158216688368056, + "matched_frac_abs_band_std": 0.003965998166427926, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/config.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f68b1fb974944b61099202c8efde7b2e80afeb45 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..de30792f81599d88619d6654bbc52d23ceda676a --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4542156a37ee3ec99e7ba415d096952c436b8464c17fed4820dd34d93c322549 +size 75550084 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..df6d9e36287b8f9b905709efa9968c3156cb932b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e903fd90d7211921b1f0515505b4937f0b2d8dd8b97c858206a70538901b7e5 +size 99648 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fbf4f81e73926065dc5461669a57588f2e66c37 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45e5038b4f48b41b4f34484b212a519f5c329d0966db5b0bfc51e8a2697abca7 +size 50471 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..71d4a2af77fac164e1a733bed74de3bdd9437c96 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.7182478515625, + "l0_per_token_0.001": 127.7182478515625, + "l0_per_token_0.01": 127.7182478515625, + "l0_per_token_0.05": 127.7182478515625, + "l0_per_token_0.1": 127.7182478515625, + "l0_per_token_0.5": 127.7182478515625, + "l0_per_token_1.0": 127.7182478515625, + "l1_per_token": 607.7213934281276, + "r2_score": 0.9548467507110822, + "l2_loss": 1.058953591401838 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..43c92ac22564999122622c8fa354bc475f3b1973 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ea7efe657c3b33aa7815ec8560dcf7665dde20243847cb0588d0c262a5dec73 +size 50382 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..339264403388301e3f509caa0435abbcfa84be09 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c339d6727326b9cbe3da1f658fed6393fe63b7604c0c5b5ca0728ca54588955 +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c146abc57a943c892b115b59dca162a16af95014 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4e75a4e30e0d7f1bc6b5b245362a9956cabe7ebddc5f93699cc42a61da926af +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/config.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7bbdb76129185355c9135442b24bb62c1df7a497 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e5e4cb544061ef090c13030ed8df45f28b88ace4 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c808be8f317e843dfd2f0bcba803f04b3317529ddb4d8f03dc89c0445b3ba893 +size 75550084 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..80470e4334d112796849549b6dba3de76d40ff66 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f729fb21572179aef4947eba759e49089330dc258139f1b0fb4f193c9db06960 +size 99648 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ec19d69a4f7afa73da173402e723b6ea1a0566f --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57b51963dd53853f0f9514151a761dffc57ee52c9858e28ae401913c0a435387 +size 50471 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..148802db6faf86eecf7ec30c6ade6d674a36a75f --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.3524513671875, + "l0_per_token_0.001": 128.3524513671875, + "l0_per_token_0.01": 128.3524513671875, + "l0_per_token_0.05": 128.3524513671875, + "l0_per_token_0.1": 128.3524513671875, + "l0_per_token_0.5": 128.3524513671875, + "l0_per_token_1.0": 128.3524513671875, + "l1_per_token": 610.2288667752479, + "r2_score": 0.9549618942987953, + "l2_loss": 1.0562531940294237 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6d061ea2cd8c37aaee346fa5ee2907fc56beca1 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9edd29217b79ee3dee47e88e274db68ffb05cdc1dc6ea9d33894806ce7eead3d +size 50382 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2aeb5a74046f74872d444a0f77cf1b220f19af2 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fde69c672847047db110cadad3443396db29fcd9613828ff9f5bda51a2914b8 +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..55a9c1697c6aa23cbedcf1c312be07bee24c50ca --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f268691c6c89f18ce1e2c30a68f361e818c37455ff20a39ad770ec1eb161695f +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/config.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..65ad07e6db1c726a7a35612ab887cd3f3b403c4c --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d264982bd115c4d452f631df590ffb8dcafba303 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81d2f343f0dbeda4e2ffa6d049714ebc2c2f45a8f8ecbffa58115ccb8c7f4dac +size 75550084 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8280bab33dd9dd0e3f2f06a4e505728f2c9ec92 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:367f01c34387df42f6df2e31a1ed498cf7936b9e4ea04da1e2cfc1f56fc4938f +size 99648 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..74cd6993d229aafa68bda5afa50f5ae2e44a732c --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78e83f048ecef584c406203f717dd5d89ea771421359414662583767ec50c008 +size 50471 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..aa682d3d762f4eace345603436611e5994846661 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.48565991210937, + "l0_per_token_0.001": 127.48565991210937, + "l0_per_token_0.01": 127.48565991210937, + "l0_per_token_0.05": 127.48565991210937, + "l0_per_token_0.1": 127.48565991210937, + "l0_per_token_0.5": 127.48565991210937, + "l0_per_token_1.0": 127.48565991210937, + "l1_per_token": 606.2793811777946, + "r2_score": 0.954758292979353, + "l2_loss": 1.061028140502446 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..10ad27685fe267bc2f760b3c891e2c70910c27af --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d02e06c8cfeed8e8ce03a03f474e41cd868e20c1b45dc50467599f5fc79fa9b8 +size 50382 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..30008f13524479e5ed93472e9e22a489f35de5ba --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cca20f1e17748b6887b333f51da83fe12658a498e6a1a0a41dc8234f91be04ac +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e48d02a78d8adefd41db8552b3d1f81b3cbca98 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61af4facba39ec6d7e84c402db4b1cfc7248324f7ac99a2476e09b071008d3b8 +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/code_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..2dfa331df81cf7ff71eafd7dc826ee3a2856690a --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 31.919266666666665, + "recon_r2": 0.9193748679709128, + "l1_per_token": 304.20424793829403 + }, + "seed1": { + "l0": 31.818753333333333, + "recon_r2": 0.9194302259933855, + "l1_per_token": 306.4840420251751 + }, + "seed2": { + "l0": 31.798905, + "recon_r2": 0.9193767591649853, + "l1_per_token": 305.6252251752166 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7400087756930157, + "mcc_abs": 0.7400210476221692, + "act_iou": 0.43364045041840676, + "act_iou_n_pairs": 12288, + "input_r2": 0.9686586865788722, + "matched_mcc_signed": { + "0.6": 0.8444406047600785, + "0.7": 0.8733211255663039, + "0.8": 0.9007055349628019, + "0.9": 0.9319587076178429 + }, + "matched_mcc_abs": { + "0.6": 0.8424924662374571, + "0.7": 0.8729844102628117, + "0.8": 0.9003978364915014, + "0.9": 0.9317171967888204 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.744693272009517, + "mcc_abs": 0.74470590875556, + "act_iou": 0.43729593880994333, + "act_iou_n_pairs": 12288, + "input_r2": 0.9695608845428498, + "matched_mcc_signed": { + "0.6": 0.8443687195173901, + "0.7": 0.8749583129128411, + "0.8": 0.9012890242036748, + "0.9": 0.9319492239243443 + }, + "matched_mcc_abs": { + "0.6": 0.8433518751414391, + "0.7": 0.8743328909963229, + "0.8": 0.9008074760540641, + "0.9": 0.9315127755633315 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.742542751517254, + "mcc_abs": 0.742554242294822, + "act_iou": 0.4358845957507067, + "act_iou_n_pairs": 12288, + "input_r2": 0.9696325740204559, + "matched_mcc_signed": { + "0.6": 0.8449652732784033, + "0.7": 0.8733104171762524, + "0.8": 0.9015445092077589, + "0.9": 0.9318724590191894 + }, + "matched_mcc_abs": { + "0.6": 0.844412736067978, + "0.7": 0.8730420062902585, + "0.8": 0.9014987154182973, + "0.9": 0.9316214859387513 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 31.845641666666666, + "l0_std": 0.06452882730049893, + "recon_r2_mean": 0.9193939510430945, + "recon_r2_std": 3.142925655682355e-05, + "l1_per_token_mean": 305.4378383795619, + "l1_per_token_std": 1.1513907364720959, + "input_r2_mean": 0.969284048380726, + "input_r2_std": 0.0005427641133247415, + "mcc_signed_mean": 0.7424149330732622, + "mcc_signed_std": 0.0023448623841916204, + "mcc_abs_mean": 0.7424270662241838, + "mcc_abs_std": 0.002345018395342776, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8445915325186241, + "0.7": 0.8738632852184658, + "0.8": 0.9011796894580786, + "0.9": 0.9319267968537922 + }, + "matched_mcc_signed_band": 0.8878903260122402, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8434190258156247, + "0.7": 0.8734531025164644, + "0.8": 0.9009013426546209, + "0.9": 0.9316171527636343 + }, + "matched_mcc_abs_band": 0.887347655937586, + "act_iou_mean": 0.43560699499301897, + "act_iou_std": 0.0018434873150272191, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/dict_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..51060367ce3dc310a008f8aaaa4da1700cf5790b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 31.96863642578125, + "recon_r2": 0.9192533434187236, + "l1_per_token": 304.34399717607556, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 20, + "dense_rate": 0.0016276041666666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1951, + "0.6": 660, + "0.7": 227, + "0.8": 138, + "0.9": 120 + }, + "n_dup_clusters_signed": { + "0.5": 877, + "0.6": 260, + "0.7": 51, + "0.8": 8, + "0.9": 7 + }, + "largest_dup_signed": { + "0.5": 128, + "0.6": 127, + "0.7": 127, + "0.8": 99, + "0.9": 51 + }, + "babel_mu1_signed": 0.40909543633461, + "n_dup_abs": { + "0.5": 2123, + "0.6": 801, + "0.7": 340, + "0.8": 230, + "0.9": 180 + }, + "n_dup_clusters_abs": { + "0.5": 959, + "0.6": 329, + "0.7": 107, + "0.8": 54, + "0.9": 37 + }, + "largest_dup_abs": { + "0.5": 128, + "0.6": 127, + "0.7": 127, + "0.8": 99, + "0.9": 51 + }, + "babel_mu1_abs": 0.41840028762817383 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 31.867031298828124, + "recon_r2": 0.9193055911896232, + "l1_per_token": 306.4091118377043, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 22, + "dense_rate": 0.0017903645833333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1930, + "0.6": 651, + "0.7": 240, + "0.8": 138, + "0.9": 122 + }, + "n_dup_clusters_signed": { + "0.5": 871, + "0.6": 260, + "0.7": 59, + "0.8": 11, + "0.9": 22 + }, + "largest_dup_signed": { + "0.5": 124, + "0.6": 124, + "0.7": 80, + "0.8": 44, + "0.9": 22 + }, + "babel_mu1_signed": 0.4086414575576782, + "n_dup_abs": { + "0.5": 2112, + "0.6": 796, + "0.7": 352, + "0.8": 232, + "0.9": 189 + }, + "n_dup_clusters_abs": { + "0.5": 957, + "0.6": 331, + "0.7": 114, + "0.8": 57, + "0.9": 55 + }, + "largest_dup_abs": { + "0.5": 124, + "0.6": 124, + "0.7": 80, + "0.8": 44, + "0.9": 22 + }, + "babel_mu1_abs": 0.4184608459472656 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 31.848321533203126, + "recon_r2": 0.9192398937198489, + "l1_per_token": 305.4881751320176, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 20, + "dense_rate": 0.0016276041666666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1969, + "0.6": 635, + "0.7": 243, + "0.8": 145, + "0.9": 132 + }, + "n_dup_clusters_signed": { + "0.5": 882, + "0.6": 245, + "0.7": 55, + "0.8": 7, + "0.9": 9 + }, + "largest_dup_signed": { + "0.5": 134, + "0.6": 134, + "0.7": 133, + "0.8": 133, + "0.9": 63 + }, + "babel_mu1_signed": 0.40897050499916077, + "n_dup_abs": { + "0.5": 2138, + "0.6": 770, + "0.7": 364, + "0.8": 245, + "0.9": 202 + }, + "n_dup_clusters_abs": { + "0.5": 964, + "0.6": 312, + "0.7": 115, + "0.8": 57, + "0.9": 44 + }, + "largest_dup_abs": { + "0.5": 134, + "0.6": 134, + "0.7": 133, + "0.8": 133, + "0.9": 63 + }, + "babel_mu1_abs": 0.41863396763801575 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8013520240783691, + "dcs_abs": 0.8026490211486816, + "frac_sign_flipped": 0.014241536458333334, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7888076901435852, + "0.6": 0.7969455718994141, + "0.7": 0.7999643683433533, + "0.8": 0.8003712296485901, + "0.9": 0.8005580902099609 + }, + "unique_frac_signed": { + "0.5": 0.9132080078125, + "0.6": 0.9678141276041666, + "0.7": 0.9854736328125, + "0.8": 0.9895426432291666, + "0.9": 0.9913330078125 + }, + "dedup_dcs_signed": { + "0.5": 0.7203454992632032, + "0.6": 0.771295183415835, + "0.7": 0.7883437921918812, + "0.8": 0.7920014621510442, + "0.9": 0.7936196594964713 + }, + "matched_frac_signed": { + "0.5": 0.7733561197916666, + "0.6": 0.7854817708333334, + "0.7": 0.7486165364583334, + "0.8": 0.6634928385416666, + "0.9": 0.5017903645833334 + }, + "matched_cos_abs": { + "0.5": 0.7907242774963379, + "0.6": 0.798725962638855, + "0.7": 0.8015782237052917, + "0.8": 0.8020060062408447, + "0.9": 0.8020365834236145 + }, + "unique_frac_abs": { + "0.5": 0.9056396484375, + "0.6": 0.9618733723958334, + "0.7": 0.9808349609375, + "0.8": 0.9857177734375, + "0.9": 0.9887288411458334 + }, + "dedup_dcs_abs": { + "0.5": 0.7161112566827796, + "0.6": 0.7682732353035439, + "0.7": 0.7862159457363305, + "0.8": 0.7905515747552272, + "0.9": 0.7929967016849939 + }, + "matched_frac_abs": { + "0.5": 0.7698567708333334, + "0.6": 0.783935546875, + "0.7": 0.746826171875, + "0.8": 0.662841796875, + "0.9": 0.5011393229166666 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8060774207115173, + "dcs_abs": 0.807426929473877, + "frac_sign_flipped": 0.014485677083333334, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7918343544006348, + "0.6": 0.8003867268562317, + "0.7": 0.8041903376579285, + "0.8": 0.8048174381256104, + "0.9": 0.8050143718719482 + }, + "unique_frac_signed": { + "0.5": 0.9120686848958334, + "0.6": 0.9678548177083334, + "0.7": 0.9851888020833334, + "0.8": 0.9890950520833334, + "0.9": 0.9903971354166666 + }, + "dedup_dcs_signed": { + "0.5": 0.7222073182735282, + "0.6": 0.7746581496176077, + "0.7": 0.7922793154042059, + "0.8": 0.7960409458804255, + "0.9": 0.7972839278712247 + }, + "matched_frac_signed": { + "0.5": 0.7779134114583334, + "0.6": 0.7908528645833334, + "0.7": 0.7503255208333334, + "0.8": 0.6700846354166666, + "0.9": 0.5111490885416666 + }, + "matched_cos_abs": { + "0.5": 0.7933586835861206, + "0.6": 0.802190899848938, + "0.7": 0.8059757351875305, + "0.8": 0.806588351726532, + "0.9": 0.8066371083259583 + }, + "unique_frac_abs": { + "0.5": 0.9048665364583334, + "0.6": 0.962158203125, + "0.7": 0.9803873697916666, + "0.8": 0.9851888020833334, + "0.9": 0.9877522786458334 + }, + "dedup_dcs_abs": { + "0.5": 0.7178837241857158, + "0.6": 0.771834554761881, + "0.7": 0.7901684311364079, + "0.8": 0.7946418120118324, + "0.9": 0.7967576417892512 + }, + "matched_frac_abs": { + "0.5": 0.7736002604166666, + "0.6": 0.7881673177083334, + "0.7": 0.7491861979166666, + "0.8": 0.6689453125, + "0.9": 0.5105794270833334 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8028092384338379, + "dcs_abs": 0.803881824016571, + "frac_sign_flipped": 0.013427734375, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7915000319480896, + "0.6": 0.7982300519943237, + "0.7": 0.8018153309822083, + "0.8": 0.8023896217346191, + "0.9": 0.8024964332580566 + }, + "unique_frac_signed": { + "0.5": 0.9126790364583334, + "0.6": 0.9682210286458334, + "0.7": 0.9849853515625, + "0.8": 0.9892171223958334, + "0.9": 0.9909261067708334 + }, + "dedup_dcs_signed": { + "0.5": 0.7223854865151225, + "0.6": 0.7728631220379611, + "0.7": 0.7897763556757127, + "0.8": 0.7937375526526012, + "0.9": 0.795214666305886 + }, + "matched_frac_signed": { + "0.5": 0.7759602864583334, + "0.6": 0.7887369791666666, + "0.7": 0.7491048177083334, + "0.8": 0.6639811197916666, + "0.9": 0.50634765625 + }, + "matched_cos_abs": { + "0.5": 0.7931916117668152, + "0.6": 0.8001823425292969, + "0.7": 0.8033261895179749, + "0.8": 0.803741455078125, + "0.9": 0.8037148118019104 + }, + "unique_frac_abs": { + "0.5": 0.9052327473958334, + "0.6": 0.9624430338541666, + "0.7": 0.9801839192708334, + "0.8": 0.9852294921875, + "0.9": 0.9881184895833334 + }, + "dedup_dcs_abs": { + "0.5": 0.7180230219310033, + "0.6": 0.7701299213804305, + "0.7": 0.7874074128946329, + "0.8": 0.7918697856366634, + "0.9": 0.7941654658934567 + }, + "matched_frac_abs": { + "0.5": 0.7728678385416666, + "0.6": 0.784912109375, + "0.7": 0.74755859375, + "0.8": 0.663818359375, + "0.9": 0.505859375 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 21, + "n_dense_std": 1.1547005383792515, + "l0_mean": 32, + "l0_std": 0.06474222029066243, + "recon_r2_mean": 0.9192662761093985, + "recon_r2_std": null, + "l1_per_token_mean": 305.4137613819325, + "l1_per_token_std": 1.034566429027056, + "babel_mu1_signed_mean": 0.40890246629714966, + "babel_mu1_signed_std": 0.00023451253954667774, + "n_dup_signed_per_threshold": { + "0.5": 1950.0, + "0.6": 648.6666666666666, + "0.7": 236.66666666666666, + "0.8": 140.33333333333334, + "0.9": 124.66666666666667 + }, + "n_dup_signed_band": 287.5833333333333, + "n_dup_signed_band_std": 1.2583057392117918, + "n_dup_clusters_signed_per_threshold": { + "0.5": 876.6666666666666, + "0.6": 255.0, + "0.7": 55.0, + "0.8": 8.666666666666666, + "0.9": 12.666666666666666 + }, + "n_dup_clusters_signed_band": 82.83333333333334, + "n_dup_clusters_signed_band_std": 4.645786621588784, + "largest_dup_signed_per_threshold": { + "0.5": 128.66666666666666, + "0.6": 128.33333333333334, + "0.7": 113.33333333333333, + "0.8": 92.0, + "0.9": 45.333333333333336 + }, + "largest_dup_signed_band": 94.75, + "largest_dup_signed_band_std": 24.724734578959588, + "dcs_signed_mean": 0.8034128944079081, + "dcs_signed_std": 0.002419843742148464, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.721646101350618, + "0.6": 0.7729388183571345, + "0.7": 0.7901331544239333, + "0.8": 0.7939266535613569, + "0.9": 0.7953727512245273 + }, + "dedup_dcs_signed_band": 0.7880928443917381, + "dedup_dcs_signed_band_std": 0.001882862504375287, + "matched_cos_signed_per_threshold": { + "0.5": 0.7907140254974365, + "0.6": 0.7985207835833231, + "0.7": 0.80199001232783, + "0.8": 0.8025260965029398, + "0.9": 0.8026896317799886 + }, + "matched_cos_signed_band": 0.8014316310485204, + "matched_cos_signed_band_std": 0.0020783429698631793, + "unique_frac_signed_per_threshold": { + "0.5": 0.9126519097222223, + "0.6": 0.9679633246527778, + "0.7": 0.9852159288194445, + "0.8": 0.9892849392361112, + "0.9": 0.9908854166666666 + }, + "unique_frac_signed_band": 0.98333740234375, + "unique_frac_signed_band_std": 0.00020345052083331483, + "matched_frac_signed_per_threshold": { + "0.5": 0.7757432725694445, + "0.6": 0.7883572048611112, + "0.7": 0.7493489583333334, + "0.8": 0.6658528645833334, + "0.9": 0.5064290364583334 + }, + "matched_frac_signed_band": 0.6774970160590279, + "matched_frac_signed_band_std": 0.0029055935077566897, + "babel_mu1_abs_mean": 0.41849836707115173, + "babel_mu1_abs_std": 0.00012127432870639753, + "n_dup_abs_per_threshold": { + "0.5": 2124.3333333333335, + "0.6": 789.0, + "0.7": 352.0, + "0.8": 235.66666666666666, + "0.9": 190.33333333333334 + }, + "n_dup_abs_band": 391.75, + "n_dup_abs_band_std": 3.774917217635375, + "n_dup_clusters_abs_per_threshold": { + "0.5": 960.0, + "0.6": 324.0, + "0.7": 112.0, + "0.8": 56.0, + "0.9": 45.333333333333336 + }, + "n_dup_clusters_abs_band": 134.33333333333334, + "n_dup_clusters_abs_band_std": 4.259792639710686, + "largest_dup_abs_per_threshold": { + "0.5": 128.66666666666666, + "0.6": 128.33333333333334, + "0.7": 113.33333333333333, + "0.8": 92.0, + "0.9": 45.333333333333336 + }, + "largest_dup_abs_band": 94.75, + "largest_dup_abs_band_std": 24.724734578959588, + "dcs_abs_mean": 0.8046525915463766, + "dcs_abs_std": 0.0024804563954531683, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.7173393342664997, + "0.6": 0.7700792371486185, + "0.7": 0.7879305965891238, + "0.8": 0.792354390801241, + "0.9": 0.7946399364559006 + }, + "dedup_dcs_abs_band": 0.786251040248721, + "dedup_dcs_abs_band_std": 0.0019454711076706708, + "matched_cos_abs_per_threshold": { + "0.5": 0.7924248576164246, + "0.6": 0.8003664016723633, + "0.7": 0.8036267161369324, + "0.8": 0.8041119376818339, + "0.9": 0.8041295011838278 + }, + "matched_cos_abs_band": 0.8030586391687393, + "matched_cos_abs_band_std": 0.0021483269535711945, + "unique_frac_abs_per_threshold": { + "0.5": 0.905246310763889, + "0.6": 0.962158203125, + "0.7": 0.98046875, + "0.8": 0.9853786892361112, + "0.9": 0.9881998697916666 + }, + "unique_frac_abs_band": 0.9790513780381944, + "unique_frac_abs_band_std": 0.00021442887035812565, + "matched_frac_abs_per_threshold": { + "0.5": 0.7721082899305555, + "0.6": 0.7856716579861112, + "0.7": 0.7478569878472222, + "0.8": 0.6652018229166666, + "0.9": 0.505859375 + }, + "matched_frac_abs_band": 0.6761474609375, + "matched_frac_abs_band_std": 0.002816963304139469, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/config.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1471f6d35c7bc8f1a7d2c3f7010e993d4c4e69ce --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c424230a7ab5923441fd3f7f310c00b631b076a6 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c6c2199c9edf2af46210a92a87cc0603b21a08c29b150f42e1fe73dbd376c48 +size 75550084 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2441979f1b7b5b3590d0c1c26126ae14ac34edcf --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2714be279f3b7a7edb76b1986f25f063d27243ad7ce03de6156cad09182cbce +size 99648 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..92dbfe6e24b4acccaf269ac76f3735067d603054 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a10b442de0cdce6f5c0c62cda2d5f28e61d69107db1738b5d27495aebed49c7 +size 50471 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..a398569aa4c25d4a9dad41945a85efaac7fc70b2 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.96863642578125, + "l0_per_token_0.001": 31.96863642578125, + "l0_per_token_0.01": 31.96863642578125, + "l0_per_token_0.05": 31.96863642578125, + "l0_per_token_0.1": 31.96863642578125, + "l0_per_token_0.5": 31.96863642578125, + "l0_per_token_1.0": 31.96863642578125, + "l1_per_token": 304.34399717607556, + "r2_score": 0.9192533434187236, + "l2_loss": 1.8937056209024563 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca8237583259afded89d7b18929fa4a3da13dae6 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8064e976734d923553dc88762712f13d7102d0158b12dee30854a7fe18a887c +size 50382 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e481a234958e835bd73af1702b5233e13759a927 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98f7dfb8b1c6e7b94e18ebf154a86be17a59bcefc099499faf1d4c8bbf808277 +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9d036a47435f4aff32b5579c94d181c58008575 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d48a6d8329da1058c73a7b0d533ba639f0e04862d2bd5217fb55ec4d605dbc05 +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/config.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..113dc9e5733aeadd53c79f20243ec2c98e11f200 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e6b9c6622355db0dbc1f09c8dc0e2720feeca3f0 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f74337b486fc1617db11683aadad32dba731d172d1f30a1bd3637b28051a396b +size 75550084 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2e777d6169b75dcffd9819d829acd9226004c09 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3d7831b6dea24ba39fd2f653f5f34031076d45bb738ef459882af4b2c7b2385 +size 99648 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..e62a0a8bdab9a0a00bd63e0ea1ec4c5623b9eb3d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08bac7104ddae5fd076e1471370107d34af9bdeb2d4d88f3716bf71a8d35748b +size 50471 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..a5246ba85aa2c09707956a4c342c73b1b95f5401 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.867031298828124, + "l0_per_token_0.001": 31.867031298828124, + "l0_per_token_0.01": 31.867031298828124, + "l0_per_token_0.05": 31.867031298828124, + "l0_per_token_0.1": 31.867031298828124, + "l0_per_token_0.5": 31.867031298828124, + "l0_per_token_1.0": 31.867031298828124, + "l1_per_token": 306.4091118377043, + "r2_score": 0.9193055911896232, + "l2_loss": 1.892480283512387 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c122804ed37354b9304fee0973f43823765303d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b26d1ce015f466b7e7b329191c8a45245802592d3642d22a21b2e784701d61b +size 50382 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9fa7ac6f415c2ea92cdc20b40ba6c64fb9c0cfb8 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:350318a196a0217fa49aea67182701743561dcb62cc32a91ab968e4bc0cdd72d +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..804d73abff2b940273bafa307c2715422d0f3f9e --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8931c98b91e66273ba09c5502ff526cb51526fa467619142f247d773eba40381 +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/config.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f96b94c5cdf87a80891a0447037ec702c16750c5 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e1b272b983f0b126e87595650bf2c9aa7d337b23 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9453a3c159240fb5b70f659b05fb55bb8ff4a6310fc8114aa7e4edbcb1a92030 +size 75550084 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb9f2bf760d7c83a0814111179590cbc4d008d88 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dae7b08234e54de922aeff1f76d8ee54a6af19f94169f9140314ec003f3953cb +size 99648 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..72b5e4e383ae6a28bb213593ff80acd5ec87be1b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ded62ecad6bfd9f434a7e990829d746b751448160bfe39ae4a00372f078129ae +size 50471 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..86162e041db7a18370fdfa8ebe3510348e69ff42 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.848321533203126, + "l0_per_token_0.001": 31.848321533203126, + "l0_per_token_0.01": 31.848321533203126, + "l0_per_token_0.05": 31.848321533203126, + "l0_per_token_0.1": 31.848321533203126, + "l0_per_token_0.5": 31.848321533203126, + "l0_per_token_1.0": 31.848321533203126, + "l1_per_token": 305.4881751320176, + "r2_score": 0.9192398937198489, + "l2_loss": 1.894021049075419 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..92735fb20b86a0cef2eaaaeaf79e9d54f8ea7203 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f36442093ae7e6833fcedda4b2db9c879238dc35b13391b5924b3f91f7c170f +size 50382 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..fad3fa144e0101686ee78af237ebffe8e726b09d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f211375a367d5e3ad7013faa5924378cf4a539674c0388c532b24199f64dac4 +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1239ed5171b43e4bfdc995e36e3b8c746c0cc6d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13b490bb56b8215ba12103d4b3ad4346e261057af6fd3abda2acc1934f78c1ad +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/code_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..52149024280bda0a57e7d92c817a1cb58b858b4b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 63.80273833333333, + "recon_r2": 0.9374612551529791, + "l1_per_token": 409.6325105573845 + }, + "seed1": { + "l0": 63.75897166666667, + "recon_r2": 0.9374669828839128, + "l1_per_token": 415.059044266032 + }, + "seed2": { + "l0": 63.48837833333333, + "recon_r2": 0.9373747606942447, + "l1_per_token": 412.39176443490027 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7280437586761858, + "mcc_abs": 0.7280614315231052, + "act_iou": 0.39033801886081143, + "act_iou_n_pairs": 12288, + "input_r2": 0.9719215081833988, + "matched_mcc_signed": { + "0.6": 0.853671209398113, + "0.7": 0.8807469125069528, + "0.8": 0.9080057330144923, + "0.9": 0.9376259228526859 + }, + "matched_mcc_abs": { + "0.6": 0.8527501765574765, + "0.7": 0.8801429246556516, + "0.8": 0.9076359927396002, + "0.9": 0.9373606164600665 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7287073347379934, + "mcc_abs": 0.7287211250080076, + "act_iou": 0.38998862640603393, + "act_iou_n_pairs": 12288, + "input_r2": 0.9725056762263585, + "matched_mcc_signed": { + "0.6": 0.8519884721966209, + "0.7": 0.8789664645316263, + "0.8": 0.9062551950754031, + "0.9": 0.9374613129418391 + }, + "matched_mcc_abs": { + "0.6": 0.8509569478347622, + "0.7": 0.8785314510386113, + "0.8": 0.9060639270373901, + "0.9": 0.9371198398859144 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7274490672525858, + "mcc_abs": 0.7274662546621175, + "act_iou": 0.3904064026568174, + "act_iou_n_pairs": 12288, + "input_r2": 0.9728142865238932, + "matched_mcc_signed": { + "0.6": 0.8516009891053272, + "0.7": 0.8802618354473376, + "0.8": 0.90688383400876, + "0.9": 0.9380179862847756 + }, + "matched_mcc_abs": { + "0.6": 0.8505509015494153, + "0.7": 0.8797558176189633, + "0.8": 0.9065177236958109, + "0.9": 0.937634645754868 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 63.68336277777778, + "l0_std": 0.17027354597605485, + "recon_r2_mean": 0.9374343329103789, + "recon_r2_std": 5.167047925809909e-05, + "l1_per_token_mean": 412.36110641943895, + "l1_per_token_std": 2.713396756503919, + "input_r2_mean": 0.9724138236445502, + "input_r2_std": 0.00045342139771146925, + "mcc_signed_mean": 0.7280667202222549, + "mcc_signed_std": 0.0006294479252939817, + "mcc_abs_mean": 0.7280829370644101, + "mcc_abs_std": 0.000627711528075361, + "matched_mcc_signed_per_threshold": { + "0.6": 0.852420223566687, + "0.7": 0.8799917374953056, + "0.8": 0.9070482540328851, + "0.9": 0.9377017406931002 + }, + "matched_mcc_signed_band": 0.8942904889469945, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8514193419805514, + "0.7": 0.8794767311044088, + "0.8": 0.9067392144909338, + "0.9": 0.937371700700283 + }, + "matched_mcc_abs_band": 0.8937517470690443, + "act_iou_mean": 0.3902443493078876, + "act_iou_std": 0.0002240864515948924, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/dict_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..de6b7a65f5997185c5c928bbeb2ff32119f7b91f --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 63.87325620117188, + "recon_r2": 0.9373691404652407, + "l1_per_token": 409.6728967933847, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 57, + "dense_rate": 0.004638671875, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 786, + "0.6": 227, + "0.7": 73, + "0.8": 52, + "0.9": 41 + }, + "n_dup_clusters_signed": { + "0.5": 362, + "0.6": 90, + "0.7": 16, + "0.8": 7, + "0.9": 6 + }, + "largest_dup_signed": { + "0.5": 42, + "0.6": 42, + "0.7": 42, + "0.8": 39, + "0.9": 25 + }, + "babel_mu1_signed": 0.3408517837524414, + "n_dup_abs": { + "0.5": 1109, + "0.6": 502, + "0.7": 310, + "0.8": 257, + "0.9": 154 + }, + "n_dup_clusters_abs": { + "0.5": 521, + "0.6": 226, + "0.7": 134, + "0.8": 109, + "0.9": 62 + }, + "largest_dup_abs": { + "0.5": 42, + "0.6": 42, + "0.7": 42, + "0.8": 39, + "0.9": 25 + }, + "babel_mu1_abs": 0.36149272322654724 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 63.804220703125, + "recon_r2": 0.9373638321279417, + "l1_per_token": 414.93116425485385, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 61, + "dense_rate": 0.004964192708333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 823, + "0.6": 227, + "0.7": 80, + "0.8": 52, + "0.9": 37 + }, + "n_dup_clusters_signed": { + "0.5": 382, + "0.6": 89, + "0.7": 20, + "0.8": 6, + "0.9": 3 + }, + "largest_dup_signed": { + "0.5": 42, + "0.6": 42, + "0.7": 42, + "0.8": 42, + "0.9": 33 + }, + "babel_mu1_signed": 0.3420127332210541, + "n_dup_abs": { + "0.5": 1147, + "0.6": 532, + "0.7": 328, + "0.8": 259, + "0.9": 153 + }, + "n_dup_clusters_abs": { + "0.5": 540, + "0.6": 239, + "0.7": 143, + "0.8": 109, + "0.9": 61 + }, + "largest_dup_abs": { + "0.5": 42, + "0.6": 42, + "0.7": 42, + "0.8": 42, + "0.9": 33 + }, + "babel_mu1_abs": 0.36312392354011536 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 63.5650908203125, + "recon_r2": 0.9372870095155903, + "l1_per_token": 412.41911611781524, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 57, + "dense_rate": 0.004638671875, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 829, + "0.6": 253, + "0.7": 81, + "0.8": 46, + "0.9": 35 + }, + "n_dup_clusters_signed": { + "0.5": 385, + "0.6": 104, + "0.7": 22, + "0.8": 5, + "0.9": 3 + }, + "largest_dup_signed": { + "0.5": 38, + "0.6": 38, + "0.7": 38, + "0.8": 38, + "0.9": 26 + }, + "babel_mu1_signed": 0.3419364392757416, + "n_dup_abs": { + "0.5": 1153, + "0.6": 531, + "0.7": 333, + "0.8": 263, + "0.9": 155 + }, + "n_dup_clusters_abs": { + "0.5": 544, + "0.6": 242, + "0.7": 147, + "0.8": 113, + "0.9": 63 + }, + "largest_dup_abs": { + "0.5": 38, + "0.6": 38, + "0.7": 38, + "0.8": 38, + "0.9": 26 + }, + "babel_mu1_abs": 0.3631896674633026 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.7609186172485352, + "dcs_abs": 0.7622218132019043, + "frac_sign_flipped": 0.017496744791666668, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7537572979927063, + "0.6": 0.7591317296028137, + "0.7": 0.7602404952049255, + "0.8": 0.7602716684341431, + "0.9": 0.760326087474823 + }, + "unique_frac_signed": { + "0.5": 0.9648030598958334, + "0.6": 0.9888102213541666, + "0.7": 0.9952392578125, + "0.8": 0.9962972005208334, + "0.9": 0.9971923828125 + }, + "dedup_dcs_signed": { + "0.5": 0.7272273475221785, + "0.6": 0.7506372135855296, + "0.7": 0.7566211862067576, + "0.8": 0.75745653489624, + "0.9": 0.7581913828835241 + }, + "matched_frac_signed": { + "0.5": 0.7914225260416666, + "0.6": 0.7627766927083334, + "0.7": 0.69775390625, + "0.8": 0.5911458333333334, + "0.9": 0.3999837239583333 + }, + "matched_cos_abs": { + "0.5": 0.756357729434967, + "0.6": 0.7618780136108398, + "0.7": 0.7622942924499512, + "0.8": 0.7621481418609619, + "0.9": 0.7618553638458252 + }, + "unique_frac_abs": { + "0.5": 0.9513753255208334, + "0.6": 0.9768473307291666, + "0.7": 0.9853108723958334, + "0.8": 0.9878743489583334, + "0.9": 0.9925130208333334 + }, + "dedup_dcs_abs": { + "0.5": 0.7195800810513902, + "0.6": 0.7442385039369885, + "0.7": 0.751096854316226, + "0.8": 0.7529065994507013, + "0.9": 0.7561513686086982 + }, + "matched_frac_abs": { + "0.5": 0.782470703125, + "0.6": 0.7571614583333334, + "0.7": 0.6935221354166666, + "0.8": 0.5882161458333334, + "0.9": 0.3983561197916667 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.7620628476142883, + "dcs_abs": 0.7636083960533142, + "frac_sign_flipped": 0.017740885416666668, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7559086680412292, + "0.6": 0.7603321671485901, + "0.7": 0.7615960836410522, + "0.8": 0.7616885304450989, + "0.9": 0.7616768479347229 + }, + "unique_frac_signed": { + "0.5": 0.9646809895833334, + "0.6": 0.9883626302083334, + "0.7": 0.9952799479166666, + "0.8": 0.9965006510416666, + "0.9": 0.9972737630208334 + }, + "dedup_dcs_signed": { + "0.5": 0.7292107219206325, + "0.6": 0.7514839005549827, + "0.7": 0.7580013104598038, + "0.8": 0.7590231164795114, + "0.9": 0.7596003363457082 + }, + "matched_frac_signed": { + "0.5": 0.793701171875, + "0.6": 0.7630208333333334, + "0.7": 0.698974609375, + "0.8": 0.5936686197916666, + "0.9": 0.3955891927083333 + }, + "matched_cos_abs": { + "0.5": 0.7586559653282166, + "0.6": 0.7627294659614563, + "0.7": 0.7638524174690247, + "0.8": 0.7634882926940918, + "0.9": 0.7633045315742493 + }, + "unique_frac_abs": { + "0.5": 0.9512939453125, + "0.6": 0.9770100911458334, + "0.7": 0.9852701822916666, + "0.8": 0.9878743489583334, + "0.9": 0.9925130208333334 + }, + "dedup_dcs_abs": { + "0.5": 0.7217048263919423, + "0.6": 0.7451943850586152, + "0.7": 0.7526010106036362, + "0.8": 0.7542305000824854, + "0.9": 0.7575896864485306 + }, + "matched_frac_abs": { + "0.5": 0.7855631510416666, + "0.6": 0.7568359375, + "0.7": 0.6946614583333334, + "0.8": 0.5906575520833334, + "0.9": 0.3939615885416667 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.7617653012275696, + "dcs_abs": 0.7632059454917908, + "frac_sign_flipped": 0.018717447916666668, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7548460364341736, + "0.6": 0.7602415680885315, + "0.7": 0.7611275911331177, + "0.8": 0.7613111734390259, + "0.9": 0.7613546848297119 + }, + "unique_frac_signed": { + "0.5": 0.9639892578125, + "0.6": 0.9883219401041666, + "0.7": 0.9951578776041666, + "0.8": 0.9964599609375, + "0.9": 0.997314453125 + }, + "dedup_dcs_signed": { + "0.5": 0.7276634704248863, + "0.6": 0.7513634215210914, + "0.7": 0.7574421181780053, + "0.8": 0.758616102146334, + "0.9": 0.7593100311351009 + }, + "matched_frac_signed": { + "0.5": 0.7908528645833334, + "0.6": 0.7669270833333334, + "0.7": 0.6962076822916666, + "0.8": 0.5909830729166666, + "0.9": 0.3973795572916667 + }, + "matched_cos_abs": { + "0.5": 0.7571483254432678, + "0.6": 0.7622172236442566, + "0.7": 0.7631126046180725, + "0.8": 0.7630661129951477, + "0.9": 0.7628694176673889 + }, + "unique_frac_abs": { + "0.5": 0.9505208333333334, + "0.6": 0.976318359375, + "0.7": 0.9849039713541666, + "0.8": 0.98779296875, + "0.9": 0.9925130208333334 + }, + "dedup_dcs_abs": { + "0.5": 0.7196852572572728, + "0.6": 0.744166669275728, + "0.7": 0.7515926348787616, + "0.8": 0.7537513411079999, + "0.9": 0.757157830230426 + }, + "matched_frac_abs": { + "0.5": 0.7828776041666666, + "0.6": 0.7603352864583334, + "0.7": 0.6922200520833334, + "0.8": 0.5879720052083334, + "0.9": 0.3956705729166667 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 58, + "n_dense_std": 2.309401076758503, + "l0_mean": 64, + "l0_std": 0.16171729592342413, + "recon_r2_mean": 0.9373399940362575, + "recon_r2_std": null, + "l1_per_token_mean": 412.34105905535125, + "l1_per_token_std": 2.630002633617813, + "babel_mu1_signed_mean": 0.34160031874974567, + "babel_mu1_signed_std": 0.0006493717526693213, + "n_dup_signed_per_threshold": { + "0.5": 812.6666666666666, + "0.6": 235.66666666666666, + "0.7": 78.0, + "0.8": 50.0, + "0.9": 37.666666666666664 + }, + "n_dup_signed_band": 100.33333333333333, + "n_dup_signed_band_std": 2.9825883613622133, + "n_dup_clusters_signed_per_threshold": { + "0.5": 376.3333333333333, + "0.6": 94.33333333333333, + "0.7": 19.333333333333332, + "0.8": 6.0, + "0.9": 4.0 + }, + "n_dup_clusters_signed_band": 30.916666666666664, + "n_dup_clusters_signed_band_std": 2.2407216099581255, + "largest_dup_signed_per_threshold": { + "0.5": 40.666666666666664, + "0.6": 40.666666666666664, + "0.7": 40.666666666666664, + "0.8": 39.666666666666664, + "0.9": 28.0 + }, + "largest_dup_signed_band": 37.25, + "largest_dup_signed_band_std": 2.384848003542364, + "dcs_signed_mean": 0.7615822553634644, + "dcs_signed_std": 0.0005936708884158223, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7280338466225658, + "0.6": 0.7511615118872013, + "0.7": 0.7573548716148556, + "0.8": 0.7583652511740285, + "0.9": 0.7590339167881112 + }, + "dedup_dcs_signed_band": 0.7564788878660492, + "dedup_dcs_signed_band_std": 0.0006738713907339867, + "matched_cos_signed_per_threshold": { + "0.5": 0.7548373341560364, + "0.6": 0.7599018216133118, + "0.7": 0.7609880566596985, + "0.8": 0.7610904574394226, + "0.9": 0.7611192067464193 + }, + "matched_cos_signed_band": 0.7607748856147131, + "matched_cos_signed_band_std": 0.0006955952200772497, + "unique_frac_signed_per_threshold": { + "0.5": 0.9644911024305557, + "0.6": 0.9884982638888888, + "0.7": 0.9952256944444443, + "0.8": 0.9964192708333334, + "0.9": 0.9972601996527778 + }, + "unique_frac_signed_band": 0.994350857204861, + "unique_frac_signed_band_std": 3.572473738899785e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.7919921875, + "0.6": 0.7642415364583334, + "0.7": 0.6976453993055555, + "0.8": 0.5919325086805555, + "0.9": 0.39765082465277773 + }, + "matched_frac_signed_band": 0.6128675672743055, + "matched_frac_signed_band_std": 5.1200591602069265e-05, + "babel_mu1_abs_mean": 0.3626021047433217, + "babel_mu1_abs_std": 0.0009613147654018808, + "n_dup_abs_per_threshold": { + "0.5": 1136.3333333333333, + "0.6": 521.6666666666666, + "0.7": 323.6666666666667, + "0.8": 259.6666666666667, + "0.9": 154.0 + }, + "n_dup_abs_band": 314.75, + "n_dup_abs_band_std": 7.893826701923472, + "n_dup_clusters_abs_per_threshold": { + "0.5": 535.0, + "0.6": 235.66666666666666, + "0.7": 141.33333333333334, + "0.8": 110.33333333333333, + "0.9": 62.0 + }, + "n_dup_clusters_abs_band": 137.33333333333331, + "n_dup_clusters_abs_band_std": 4.289036410819257, + "largest_dup_abs_per_threshold": { + "0.5": 40.666666666666664, + "0.6": 40.666666666666664, + "0.7": 40.666666666666664, + "0.8": 39.666666666666664, + "0.9": 28.0 + }, + "largest_dup_abs_band": 37.25, + "largest_dup_abs_band_std": 2.384848003542364, + "dcs_abs_mean": 0.7630120515823364, + "dcs_abs_std": 0.00071333662253429, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.7203233882335351, + "0.6": 0.7445331860904439, + "0.7": 0.7517634999328746, + "0.8": 0.7536294802137289, + "0.9": 0.756966295095885 + }, + "dedup_dcs_abs_band": 0.7517231153332331, + "dedup_dcs_abs_band_std": 0.0006545807992119252, + "matched_cos_abs_per_threshold": { + "0.5": 0.7573873400688171, + "0.6": 0.7622749010721842, + "0.7": 0.7630864381790161, + "0.8": 0.7629008491834005, + "0.9": 0.7626764376958212 + }, + "matched_cos_abs_band": 0.7627346565326055, + "matched_cos_abs_band_std": 0.0006537007852029789, + "unique_frac_abs_per_threshold": { + "0.5": 0.9510633680555557, + "0.6": 0.9767252604166666, + "0.7": 0.9851616753472222, + "0.8": 0.9878472222222223, + "0.9": 0.9925130208333334 + }, + "unique_frac_abs_band": 0.9855617947048612, + "unique_frac_abs_band_std": 0.0001563836338650627, + "matched_frac_abs_per_threshold": { + "0.5": 0.7836371527777777, + "0.6": 0.7581108940972223, + "0.7": 0.6934678819444445, + "0.8": 0.5889485677083334, + "0.9": 0.39599609375 + }, + "matched_frac_abs_band": 0.6091308593750001, + "matched_frac_abs_band_std": 0.0001588999364401558, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/config.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..10dca0d51af6f32bd6a0e03266fe81f53950f413 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b11e28b71e23f3b392082c17b37f5d3e2e5550fe --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ac92c3119c2dd6ece8bcff5119fcbe070bd1e5ec9cefe2244b057b988110c1f +size 75550084 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a31c4a09e9cc97d9d2f71196652fb198723587e0 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81b39b1862f61246242a410915775c6c09e6e05618663740b0117eb18f877367 +size 99648 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..aec64a81cc44e1e742ae656cf7df6e1beed580a2 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64086cf27669439a8aecc8b8c1c52f0ba398472f89d110c345c457f7fd851ba0 +size 50471 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..ea58c33bab2895c49b0b717a638f0f9515e8c0fe --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.87325620117188, + "l0_per_token_0.001": 63.87325620117188, + "l0_per_token_0.01": 63.87325620117188, + "l0_per_token_0.05": 63.87325620117188, + "l0_per_token_0.1": 63.87325620117188, + "l0_per_token_0.5": 63.87325620117188, + "l0_per_token_1.0": 63.87325620117188, + "l1_per_token": 409.6728967933847, + "r2_score": 0.9373691404652407, + "l2_loss": 1.4688460892934097 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..eea2e9572e4e99b18ce75c4752af6b9c14881fea --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90ebec5d004cec4f19c26b413a51a6475c7218b18581cbaba821973eac548f2a +size 50382 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dbe621084ade7b0e4e7eeb0918a8d632e4c5bf03 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42c66d508b20fc666c80eb4c56c4c93005dffa7e1ab72310b9afa29a9bf56fec +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7a6cd6ea855aad804b68c152cf18137dadcc14f --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:745f2e7c06d30bec5360fb20a017ce1a22621ce5cda30a1400422a3e8a782e98 +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/config.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..854438ca02d0ba74a3c53bd7168fcd09f7a150fb --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7b966712a7f75c1a4e121daa40884f68df763a42 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38db9e8662b9481cd8d3923f923c1ea8611fe3165a3fe47351f78ac8d186b63f +size 75550084 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dfea3c678912147d142c26a560bea776792deff7 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fb0fe14f775c9a0c59f55870e0018853ca23b19d6e0c538002b5326a7f93ce3 +size 99648 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..9498f8d833ec6c48cd45d2f960a1dd72a437c176 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d3994536f1413f5cbccc7741d28ef7c243d3a7895fb61e942055605147d08b7 +size 50471 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..0de248006fa35e8a379ba2479e773520e4ffdccc --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.804220703125, + "l0_per_token_0.001": 63.804220703125, + "l0_per_token_0.01": 63.804220703125, + "l0_per_token_0.05": 63.804220703125, + "l0_per_token_0.1": 63.804220703125, + "l0_per_token_0.5": 63.804220703125, + "l0_per_token_1.0": 63.804220703125, + "l1_per_token": 414.93116425485385, + "r2_score": 0.9373638321279417, + "l2_loss": 1.4689705827226918 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e5c84131274c7afbb79524198e453f56c2b329d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97ffbf4db6fe1b6d94798bf74fdfabcc4b8232a045ded77c8b10efef9046d0fd +size 50382 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..db29903179b087c79e3b40bdcd0da3a0faf9f19f --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f16bd420488ae0b5fcf1e6574f3e1562dc23392e2e0505d2fe799e512fd5816 +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a8898550640ea77b6cbfdd3f37910590dc88d24 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:129c77f7ece3af8d6c30b44895f36087756411a1bdf4cee6b9d21b81a445ceb2 +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/config.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f09a7c503cce76d640a14afb82902f286199463c --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..84a2e917fd48c00ccd4f59ed3b8085e9e80ed133 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:327311ce5a7b006b58dc9beabc6038ba0024130552208a49ff645176e8cc5f9e +size 75550084 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5e06a7b73d8764e34217fee30c5cf0baeadfdfd --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3f4bf62b2356f9b9dead36228889e20507098a4c26079328124482ad54b9af6 +size 99648 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..956294fa8fbd0dd65f6995c9f47e09e917766d35 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e02bfc2b541b281a365352521e5c8fc6953158323d2c9f3898fc137bf4f26f17 +size 50471 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..a718d2d3ba183852cd75fbb8c4d285c2d81b0a4c --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.5650908203125, + "l0_per_token_0.001": 63.5650908203125, + "l0_per_token_0.01": 63.5650908203125, + "l0_per_token_0.05": 63.5650908203125, + "l0_per_token_0.1": 63.5650908203125, + "l0_per_token_0.5": 63.5650908203125, + "l0_per_token_1.0": 63.5650908203125, + "l1_per_token": 412.41911611781524, + "r2_score": 0.9372870095155903, + "l2_loss": 1.470772259955924 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2db673cf165142739bd837293ee16d2f2ef69828 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80f515f8109a5d735be1e9c65b1414cde62daa2859b3f764966b1a8db32ce980 +size 50382 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba19bde57c1d3c9d3132001f22e7e24e97afa47b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3282ef671b2a6ee2ed19fa421f96fe04eb6d7a6bf90a8aaf49020bd8d1e9f0e +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea2f9ee871f5bab88c202fd66c01be93811f3688 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:042835e5b7a29319f83463a9c363729abc9b3af4d4d6ced7a9c79f296b555dcb +size 50387 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/code_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..22f30d8035c725f09708a709f24cfedf4caf1b58 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 127.7453, + "recon_r2": 0.9420004992500728, + "l1_per_token": 658.4036728899638 + }, + "seed1": { + "l0": 127.84756333333333, + "recon_r2": 0.9419144025192066, + "l1_per_token": 661.196970222557 + }, + "seed2": { + "l0": 127.58651, + "recon_r2": 0.9419541135860873, + "l1_per_token": 655.8631977099268 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7273289710754748, + "mcc_abs": 0.727392388071802, + "act_iou": 0.4160458508374725, + "act_iou_n_pairs": 3072, + "input_r2": 0.9621860779526052, + "matched_mcc_signed": { + "0.6": 0.8757908801480867, + "0.7": 0.8993815356573047, + "0.8": 0.9229326328666666, + "0.9": 0.9529614092158369 + }, + "matched_mcc_abs": { + "0.6": 0.8739588097365099, + "0.7": 0.8980983499495632, + "0.8": 0.9212844973596555, + "0.9": 0.9513589349138927 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7249561325830562, + "mcc_abs": 0.7250401240840564, + "act_iou": 0.4133309758659929, + "act_iou_n_pairs": 3072, + "input_r2": 0.9620366364056746, + "matched_mcc_signed": { + "0.6": 0.8726159590111557, + "0.7": 0.8980595695761472, + "0.8": 0.9223451427263493, + "0.9": 0.9521435447784625 + }, + "matched_mcc_abs": { + "0.6": 0.8703709059938086, + "0.7": 0.8965515331418813, + "0.8": 0.9210885542605267, + "0.9": 0.9513080245854463 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7289302419596112, + "mcc_abs": 0.7290339834461784, + "act_iou": 0.4176294657034217, + "act_iou_n_pairs": 3072, + "input_r2": 0.962251357712312, + "matched_mcc_signed": { + "0.6": 0.8786083023122164, + "0.7": 0.898599047279098, + "0.8": 0.9230823037764196, + "0.9": 0.9526946746731664 + }, + "matched_mcc_abs": { + "0.6": 0.875598195719829, + "0.7": 0.896987118142833, + "0.8": 0.9214731077908831, + "0.9": 0.9508427999523593 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 127.72645777777778, + "l0_std": 0.1315427030086177, + "recon_r2_mean": 0.9419563384517889, + "recon_r2_std": 4.30914642012301e-05, + "l1_per_token_mean": 658.4879469408157, + "l1_per_token_std": 2.6678847221830124, + "input_r2_mean": 0.9621580240235307, + "input_r2_std": 0.00011007532458915152, + "mcc_signed_mean": 0.727071781872714, + "mcc_signed_std": 0.001999498949416584, + "mcc_abs_mean": 0.7271554985340122, + "mcc_abs_std": 0.0020074400715682987, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8756717138238196, + "0.7": 0.8986800508375166, + "0.8": 0.9227866931231451, + "0.9": 0.9525998762224885 + }, + "matched_mcc_signed_band": 0.9124345835017424, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8733093038167158, + "0.7": 0.8972123337447592, + "0.8": 0.9212820531370217, + "0.9": 0.9511699198172328 + }, + "matched_mcc_abs_band": 0.9107434026289324, + "act_iou_mean": 0.415668764135629, + "act_iou_std": 0.0021739134081536168, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/dict_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..d06db52c9435a42a4f94f80226db1d0eec5a6b55 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 128, + "l0_raw": 127.82932724609375, + "recon_r2": 0.9419289688364122, + "l1_per_token": 658.2898242841162, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 293, + "dense_rate": 0.09537760416666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 42, + "0.6": 12, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "n_dup_clusters_signed": { + "0.5": 21, + "0.6": 6, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_signed": 0.21757502853870392, + "n_dup_abs": { + "0.5": 547, + "0.6": 501, + "0.7": 477, + "0.8": 467, + "0.9": 433 + }, + "n_dup_clusters_abs": { + "0.5": 273, + "0.6": 250, + "0.7": 238, + "0.8": 233, + "0.9": 216 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 3 + }, + "babel_mu1_abs": 0.36598697304725647 + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 128, + "l0_raw": 127.91179345703125, + "recon_r2": 0.94184165026627, + "l1_per_token": 661.0160904729009, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 298, + "dense_rate": 0.09700520833333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 24, + "0.6": 8, + "0.7": 2, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 12, + "0.6": 4, + "0.7": 1, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.21476681530475616, + "n_dup_abs": { + "0.5": 496, + "0.6": 456, + "0.7": 442, + "0.8": 430, + "0.9": 398 + }, + "n_dup_clusters_abs": { + "0.5": 248, + "0.6": 228, + "0.7": 221, + "0.8": 215, + "0.9": 199 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.35487303137779236 + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 128, + "l0_raw": 127.68618706054687, + "recon_r2": 0.9418873582019351, + "l1_per_token": 655.8377430833842, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 296, + "dense_rate": 0.09635416666666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 37, + "0.6": 12, + "0.7": 2, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 18, + "0.6": 6, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.21586613357067108, + "n_dup_abs": { + "0.5": 526, + "0.6": 479, + "0.7": 459, + "0.8": 450, + "0.9": 430 + }, + "n_dup_clusters_abs": { + "0.5": 262, + "0.6": 239, + "0.7": 229, + "0.8": 225, + "0.9": 215 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3609427511692047 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.7058725953102112, + "dcs_abs": 0.7077875137329102, + "frac_sign_flipped": 0.040690104166666664, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7061142921447754, + "0.6": 0.7058769464492798, + "0.7": 0.7060529589653015, + "0.8": 0.7060529589653015, + "0.9": 0.7060719132423401 + }, + "unique_frac_signed": { + "0.5": 0.99462890625, + "0.6": 0.9983723958333334, + "0.7": 0.9996744791666666, + "0.8": 0.9996744791666666, + "0.9": 0.9998372395833334 + }, + "dedup_dcs_signed": { + "0.5": 0.7023216860834509, + "0.6": 0.704728058190085, + "0.7": 0.7058231240177216, + "0.8": 0.7058231240177216, + "0.9": 0.7059569926835442 + }, + "matched_frac_signed": { + "0.5": 0.7682291666666666, + "0.6": 0.7109375, + "0.7": 0.6383463541666666, + "0.8": 0.5240885416666666, + "0.9": 0.2763671875 + }, + "matched_cos_abs": { + "0.5": 0.7295053601264954, + "0.6": 0.7288246154785156, + "0.7": 0.7288497090339661, + "0.8": 0.7287100553512573, + "0.9": 0.7270337343215942 + }, + "unique_frac_abs": { + "0.5": 0.9150390625, + "0.6": 0.9220377604166666, + "0.7": 0.9251302083333334, + "0.8": 0.9269205729166666, + "0.9": 0.9322916666666666 + }, + "dedup_dcs_abs": { + "0.5": 0.6675259008188732, + "0.6": 0.6720038161923487, + "0.7": 0.6742808831622824, + "0.8": 0.6754563419963233, + "0.9": 0.6778074918935696 + }, + "matched_frac_abs": { + "0.5": 0.736328125, + "0.6": 0.6875, + "0.7": 0.6217447916666666, + "0.8": 0.5126953125, + "0.9": 0.2698567708333333 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.7019783854484558, + "dcs_abs": 0.7041216492652893, + "frac_sign_flipped": 0.046549479166666664, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7005414962768555, + "0.6": 0.7017579674720764, + "0.7": 0.7022120952606201, + "0.8": 0.7022120952606201, + "0.9": 0.7022120952606201 + }, + "unique_frac_signed": { + "0.5": 0.9934895833333334, + "0.6": 0.998046875, + "0.7": 0.9996744791666666, + "0.8": 0.9998372395833334, + "0.9": 0.9998372395833334 + }, + "dedup_dcs_signed": { + "0.5": 0.695980679243803, + "0.6": 0.7003873464418575, + "0.7": 0.7019835105941941, + "0.8": 0.7020978029274072, + "0.9": 0.7020978029274072 + }, + "matched_frac_signed": { + "0.5": 0.7630208333333334, + "0.6": 0.7106119791666666, + "0.7": 0.62890625, + "0.8": 0.5166015625, + "0.9": 0.2662760416666667 + }, + "matched_cos_abs": { + "0.5": 0.7228727340698242, + "0.6": 0.7241484522819519, + "0.7": 0.7243272066116333, + "0.8": 0.7238067388534546, + "0.9": 0.7220762968063354 + }, + "unique_frac_abs": { + "0.5": 0.9124348958333334, + "0.6": 0.9200846354166666, + "0.7": 0.9236653645833334, + "0.8": 0.92529296875, + "0.9": 0.9296875 + }, + "dedup_dcs_abs": { + "0.5": 0.659574307811757, + "0.6": 0.6662778647053831, + "0.7": 0.6690359533725617, + "0.8": 0.669733286194969, + "0.9": 0.67130530718714 + }, + "matched_frac_abs": { + "0.5": 0.7330729166666666, + "0.6": 0.6858723958333334, + "0.7": 0.6119791666666666, + "0.8": 0.5045572916666666, + "0.9": 0.2607421875 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.7059246897697449, + "dcs_abs": 0.7083463668823242, + "frac_sign_flipped": 0.039388020833333336, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7055637836456299, + "0.6": 0.7062199711799622, + "0.7": 0.7061080932617188, + "0.8": 0.7061080932617188, + "0.9": 0.7059246897697449 + }, + "unique_frac_signed": { + "0.5": 0.9949544270833334, + "0.6": 0.9983723958333334, + "0.7": 0.9996744791666666, + "0.8": 0.9998372395833334, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7020038101278867, + "0.6": 0.7050705246122865, + "0.7": 0.7058782403667767, + "0.8": 0.7059931668142478, + "0.9": 0.7059246897697449 + }, + "matched_frac_signed": { + "0.5": 0.7639973958333334, + "0.6": 0.7112630208333334, + "0.7": 0.6461588541666666, + "0.8": 0.52734375, + "0.9": 0.275390625 + }, + "matched_cos_abs": { + "0.5": 0.7278327345848083, + "0.6": 0.7285890579223633, + "0.7": 0.728736400604248, + "0.8": 0.7285932898521423, + "0.9": 0.7278349995613098 + }, + "unique_frac_abs": { + "0.5": 0.9166666666666666, + "0.6": 0.923828125, + "0.7": 0.9265950520833334, + "0.8": 0.9283854166666666, + "0.9": 0.9326171875 + }, + "dedup_dcs_abs": { + "0.5": 0.6671800067027409, + "0.6": 0.6730910632759333, + "0.7": 0.6752435430729141, + "0.8": 0.6764153849799186, + "0.9": 0.6787914302549325 + }, + "matched_frac_abs": { + "0.5": 0.7350260416666666, + "0.6": 0.6897786458333334, + "0.7": 0.6302083333333334, + "0.8": 0.5159505208333334, + "0.9": 0.2701822916666667 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 296, + "n_dense_std": 2.516611478423583, + "l0_mean": 128, + "l0_std": 0.11415489097380761, + "recon_r2_mean": 0.9418859924348725, + "recon_r2_std": null, + "l1_per_token_mean": 658.3812192801338, + "l1_per_token_std": 2.5903832312443207, + "babel_mu1_signed_mean": 0.2160693258047104, + "babel_mu1_signed_std": 0.0014150903521719872, + "n_dup_signed_per_threshold": { + "0.5": 34.333333333333336, + "0.6": 10.666666666666666, + "0.7": 2.0, + "0.8": 1.3333333333333333, + "0.9": 0.6666666666666666 + }, + "n_dup_signed_band": 3.6666666666666665, + "n_dup_signed_band_std": 0.7637626158259733, + "n_dup_clusters_signed_per_threshold": { + "0.5": 17.0, + "0.6": 5.333333333333333, + "0.7": 1.0, + "0.8": 0.6666666666666666, + "0.9": 0.3333333333333333 + }, + "n_dup_clusters_signed_band": 1.8333333333333333, + "n_dup_clusters_signed_band_std": 0.38188130791298663, + "largest_dup_signed_per_threshold": { + "0.5": 2.3333333333333335, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 1.6666666666666667, + "0.9": 1.3333333333333333 + }, + "largest_dup_signed_band": 1.75, + "largest_dup_signed_band_std": 0.25, + "dcs_signed_mean": 0.7045918901761373, + "dcs_signed_std": 0.0022635113605609766, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7001020584850469, + "0.6": 0.7033953097480764, + "0.7": 0.7045616249928974, + "0.8": 0.7046380312531255, + "0.9": 0.704659828460232 + }, + "dedup_dcs_signed_band": 0.7043136986135828, + "dedup_dcs_signed_band_std": 0.002315058939502015, + "matched_cos_signed_per_threshold": { + "0.5": 0.7040731906890869, + "0.6": 0.7046182950337728, + "0.7": 0.7047910491625468, + "0.8": 0.7047910491625468, + "0.9": 0.7047362327575684 + }, + "matched_cos_signed_band": 0.7047341565291086, + "matched_cos_signed_band_std": 0.002282811299518657, + "unique_frac_signed_per_threshold": { + "0.5": 0.994357638888889, + "0.6": 0.998263888888889, + "0.7": 0.9996744791666666, + "0.8": 0.9997829861111112, + "0.9": 0.9998914930555557 + }, + "unique_frac_signed_band": 0.9994032118055557, + "unique_frac_signed_band_std": 6.215516079313338e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.7650824652777778, + "0.6": 0.7109375, + "0.7": 0.6378038194444443, + "0.8": 0.5226779513888888, + "0.9": 0.2726779513888889 + }, + "matched_frac_signed_band": 0.5360243055555555, + "matched_frac_signed_band_std": 0.004875573337383054, + "babel_mu1_abs_mean": 0.36060091853141785, + "babel_mu1_abs_std": 0.005564850584000879, + "n_dup_abs_per_threshold": { + "0.5": 523.0, + "0.6": 478.6666666666667, + "0.7": 459.3333333333333, + "0.8": 449.0, + "0.9": 420.3333333333333 + }, + "n_dup_abs_band": 451.8333333333333, + "n_dup_abs_band_std": 19.139836293274122, + "n_dup_clusters_abs_per_threshold": { + "0.5": 261.0, + "0.6": 239.0, + "0.7": 229.33333333333334, + "0.8": 224.33333333333334, + "0.9": 210.0 + }, + "n_dup_clusters_abs_band": 225.66666666666669, + "n_dup_clusters_abs_band_std": 9.32179346120334, + "largest_dup_abs_per_threshold": { + "0.5": 2.6666666666666665, + "0.6": 2.6666666666666665, + "0.7": 2.6666666666666665, + "0.8": 2.3333333333333335, + "0.9": 2.3333333333333335 + }, + "largest_dup_abs_band": 2.5, + "largest_dup_abs_band_std": 0.5, + "dcs_abs_mean": 0.7067518432935079, + "dcs_abs_std": 0.0022948899059018154, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6647600717777904, + "0.6": 0.6704575813912217, + "0.7": 0.6728534598692528, + "0.8": 0.673868337723737, + "0.9": 0.675968076445214 + }, + "dedup_dcs_abs_band": 0.6732868638573564, + "dedup_dcs_abs_band_std": 0.003670327946622459, + "matched_cos_abs_per_threshold": { + "0.5": 0.7267369429270426, + "0.6": 0.7271873752276102, + "0.7": 0.7273044387499491, + "0.8": 0.727036694685618, + "0.9": 0.7256483435630798 + }, + "matched_cos_abs_band": 0.7267942130565643, + "matched_cos_abs_band_std": 0.0027755296465210686, + "unique_frac_abs_per_threshold": { + "0.5": 0.9147135416666666, + "0.6": 0.9219835069444443, + "0.7": 0.9251302083333334, + "0.8": 0.9268663194444443, + "0.9": 0.9315321180555555 + }, + "unique_frac_abs_band": 0.9263780381944443, + "unique_frac_abs_band_std": 0.0015980042280101287, + "matched_frac_abs_per_threshold": { + "0.5": 0.7348090277777777, + "0.6": 0.687717013888889, + "0.7": 0.6213107638888888, + "0.8": 0.5110677083333334, + "0.9": 0.2669270833333333 + }, + "matched_frac_abs_band": 0.5217556423611112, + "matched_frac_abs_band_std": 0.005469654149795386, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/config.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c35a8861a4210a98ca2e367ff6d4aa8deaefeb55 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1762e7aa5e40fb4ad3b2b05ab655ab5a08515e43 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:289461f0b26dc3ae42722948cbe343e80b748172b355aad54e0a1ec8ecc7bee5 +size 18890116 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd943b3ec16eb9eae6fb044442f686eac05af82b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bf5c49f3457fd954cbfd0f4d27135071b41835354a378ee0b0de55fb8424bb2 +size 25920 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..d05fd23737d2be7a07121405973984960e5430af --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac5fc1f619a215db8e9f5c2bf8da4c3f6d66b29a4d9092fa9d334fc4e455e333 +size 13607 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..074dcef4db90c8d72d5ecf152a98f5fffec947a3 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.82932724609375, + "l0_per_token_0.001": 127.82932724609375, + "l0_per_token_0.01": 127.82932724609375, + "l0_per_token_0.05": 127.82932724609375, + "l0_per_token_0.1": 127.82932724609375, + "l0_per_token_0.5": 127.82932724609375, + "l0_per_token_1.0": 127.82932724609375, + "l1_per_token": 658.2898242841162, + "r2_score": 0.9419289688364122, + "l2_loss": 1.3619070161177145 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4e50f376b172dd812085154fd4efd4c6a10d558 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2a53fc4c29d5199fe663e0d3206c922db344e2a99c1097fd9d667a298afdea6 +size 13518 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a6c6b43a0353c1b54a5e540f49e93cd06a414dd --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad678c4290e7ea2a1f66e4636598d5c77a407a9ced0e218ecf3d5ef868c29006 +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..47b700ad7b368844e4fd7f6e55674f2b94a842c7 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dad6add2047bcdae25a6deee4d80d85f86061b1a1f3ec36063d095298af97bd9 +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/config.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..69eb8061c95e13eabf11b5605ce9e0273fa5b6ee --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..63581af03bea8f4edd2f83e444e17093437dd739 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c1cd76b2e067a08e952ba18c1955462c339c8b20681fc13c97cdde1e0d55fff +size 18890116 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..39944190c82bc1241e417e3c8475c3368d753c5a --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d0fbffce4c23f562e6ebb5554045b9f42f1372114e9bfba9bedc4fb8574210b +size 25920 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b4454441887bd810aaba0c3b79f3c4ad59cd0ac --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:774b59c3f9336afddca4af43818338554e57ff9c0cc94ac73553f5a319a93d41 +size 13607 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..57717abdaa269d3654ab0c346db0df8f02f9d001 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.91179345703125, + "l0_per_token_0.001": 127.91179345703125, + "l0_per_token_0.01": 127.91179345703125, + "l0_per_token_0.05": 127.91179345703125, + "l0_per_token_0.1": 127.91179345703125, + "l0_per_token_0.5": 127.91179345703125, + "l0_per_token_1.0": 127.91179345703125, + "l1_per_token": 661.0160904729009, + "r2_score": 0.94184165026627, + "l2_loss": 1.3639548491065758 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..de345d68bf5d2a8cba5adc0f826a37fecaf2516f --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5926572537f201907e0354bcb23c8cdc127aa4e67e3f88517efbf708c3e9e6e +size 13518 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a4c5845aa645fe68155ed0df46390729a0e4754 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab083b7c39bf213a4afc3cf0f3216ed243cb34d131020d190d3603a90c90127d +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6891d14131ef0a1c3824dc4009b394232aeec87 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e9d4b1d27e6804035b7b42fb057e53ae5aa6a473c70c90410e2525f8d15e574 +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/config.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ea55bca5defbb5c0d14766fcc72e07889904e940 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ce7f87b3245c50f27de68e4ab20556f73571d894 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b1e51b7cb3ae711369f04156d03712d2da7bf0a3a5b0042155d07256c1c266f +size 18890116 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0289db253e37767e38ecf73663f7abcc7556f36d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d20dd47c13da8cd5781a4edbd1fbff8fb38ff942edc1b284e2f37720dde2e1d6 +size 25920 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c709d3dec449b7a3003cb9663a3ab72b5d2a862 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7198332c7274fdcc3479f0e0c95356a208bd983f929d4086ccfa306e5ecf413 +size 13607 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..8527c359dddacc168b2f51d9412da7dbbc35c165 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.68618706054687, + "l0_per_token_0.001": 127.68618706054687, + "l0_per_token_0.01": 127.68618706054687, + "l0_per_token_0.05": 127.68618706054687, + "l0_per_token_0.1": 127.68618706054687, + "l0_per_token_0.5": 127.68618706054687, + "l0_per_token_1.0": 127.68618706054687, + "l1_per_token": 655.8377430833842, + "r2_score": 0.9418873582019351, + "l2_loss": 1.3628828867696363 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b66c40694b3503ad08cdf2a09339790dfa12b35 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b58a5cc6b017782a23bd84d8c6376694c799bcfbc5081b31c34b3532bb4c867 +size 13518 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..190bf6bb5ed95adacb872d67622b1e4e9dcd1882 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3d7c8e07acd209d52b50489f92e926c8db2a072af106a8ce73c1e7f30745167 +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..600992ef1a93956331e25f4f162b66384755d093 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e0ae86d882cbcced63cf939788c3cbc69f4949e3f1c66734c18b455793ccfc7 +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/code_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..6f0436926aff8a291517607a6cc95ca99567a020 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 31.957463333333333, + "recon_r2": 0.896220214694665, + "l1_per_token": 325.0614623816443 + }, + "seed1": { + "l0": 31.900861666666668, + "recon_r2": 0.8962228198739468, + "l1_per_token": 322.85385896545887 + }, + "seed2": { + "l0": 31.757466666666666, + "recon_r2": 0.8960450981682486, + "l1_per_token": 324.1896847629515 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7947363322618846, + "mcc_abs": 0.7947802036360363, + "act_iou": 0.5131427222105843, + "act_iou_n_pairs": 3072, + "input_r2": 0.9657613217616919, + "matched_mcc_signed": { + "0.6": 0.8823899449844077, + "0.7": 0.8988967183727878, + "0.8": 0.9153274644657584, + "0.9": 0.9367250759738897 + }, + "matched_mcc_abs": { + "0.6": 0.8809101302402065, + "0.7": 0.8977057561386923, + "0.8": 0.9149609004490298, + "0.9": 0.9365929476442919 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7912772633326144, + "mcc_abs": 0.791323303431892, + "act_iou": 0.5093945555310672, + "act_iou_n_pairs": 3072, + "input_r2": 0.9663091467323973, + "matched_mcc_signed": { + "0.6": 0.8799271658505441, + "0.7": 0.897396295407143, + "0.8": 0.914503663862777, + "0.9": 0.9362435273867299 + }, + "matched_mcc_abs": { + "0.6": 0.876556009308687, + "0.7": 0.8965603406979135, + "0.8": 0.913814827940709, + "0.9": 0.9360073060562034 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7983736361911803, + "mcc_abs": 0.7984054149924376, + "act_iou": 0.515675636717539, + "act_iou_n_pairs": 3072, + "input_r2": 0.9666031875391775, + "matched_mcc_signed": { + "0.6": 0.8814785975695064, + "0.7": 0.8992828159510962, + "0.8": 0.914692331318551, + "0.9": 0.935725845028547 + }, + "matched_mcc_abs": { + "0.6": 0.8795482177222389, + "0.7": 0.8976450694770318, + "0.8": 0.9139038704373867, + "0.9": 0.9352152689607326 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 31.871930555555554, + "l0_std": 0.10308939597280187, + "recon_r2_mean": 0.8961627109122867, + "recon_r2_std": 0.00010186395294663412, + "l1_per_token_mean": 324.0350020366849, + "l1_per_token_std": 1.1119007465337574, + "input_r2_mean": 0.9662245520110889, + "input_r2_std": 0.00042726068972031694, + "mcc_signed_mean": 0.7947957439285598, + "mcc_signed_std": 0.0035485594607601745, + "mcc_abs_mean": 0.7948363073534552, + "mcc_abs_std": 0.003541389100245075, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8812652361348193, + "0.7": 0.898525276577009, + "0.8": 0.9148411532156954, + "0.9": 0.9362314827963889 + }, + "matched_mcc_signed_band": 0.9077157871809781, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8790047857570441, + "0.7": 0.8973037221045459, + "0.8": 0.9142265329423752, + "0.9": 0.9359385075537426 + }, + "matched_mcc_abs_band": 0.906618387089427, + "act_iou_mean": 0.5127376381530635, + "act_iou_std": 0.003160073581106275, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/dict_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..201c8b5ac08461125c3a90367a5f7b50abf57003 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 32, + "l0_raw": 31.990659716796873, + "recon_r2": 0.8961075294995073, + "l1_per_token": 324.9382562512766, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 18, + "dense_rate": 0.005859375, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 204, + "0.6": 55, + "0.7": 11, + "0.8": 5, + "0.9": 3 + }, + "n_dup_clusters_signed": { + "0.5": 99, + "0.6": 27, + "0.7": 5, + "0.8": 2, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 3 + }, + "babel_mu1_signed": 0.322672039270401, + "n_dup_abs": { + "0.5": 258, + "0.6": 96, + "0.7": 42, + "0.8": 29, + "0.9": 19 + }, + "n_dup_clusters_abs": { + "0.5": 125, + "0.6": 47, + "0.7": 20, + "0.8": 14, + "0.9": 9 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 3 + }, + "babel_mu1_abs": 0.33921119570732117 + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 32, + "l0_raw": 31.92978974609375, + "recon_r2": 0.8961027107039853, + "l1_per_token": 322.7108991769086, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 19, + "dense_rate": 0.006184895833333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 165, + "0.6": 55, + "0.7": 14, + "0.8": 4, + "0.9": 4 + }, + "n_dup_clusters_signed": { + "0.5": 79, + "0.6": 26, + "0.7": 6, + "0.8": 1, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 4, + "0.6": 4, + "0.7": 4, + "0.8": 4, + "0.9": 4 + }, + "babel_mu1_signed": 0.32154691219329834, + "n_dup_abs": { + "0.5": 221, + "0.6": 106, + "0.7": 52, + "0.8": 32, + "0.9": 20 + }, + "n_dup_clusters_abs": { + "0.5": 106, + "0.6": 51, + "0.7": 25, + "0.8": 15, + "0.9": 9 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 4, + "0.7": 4, + "0.8": 4, + "0.9": 4 + }, + "babel_mu1_abs": 0.3382156193256378 + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 32, + "l0_raw": 31.7940185546875, + "recon_r2": 0.8959210819504068, + "l1_per_token": 324.0731302088276, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 17, + "dense_rate": 0.005533854166666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 177, + "0.6": 34, + "0.7": 9, + "0.8": 5, + "0.9": 3 + }, + "n_dup_clusters_signed": { + "0.5": 86, + "0.6": 16, + "0.7": 4, + "0.8": 2, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 3 + }, + "babel_mu1_signed": 0.3203188180923462, + "n_dup_abs": { + "0.5": 236, + "0.6": 77, + "0.7": 36, + "0.8": 25, + "0.9": 15 + }, + "n_dup_clusters_abs": { + "0.5": 115, + "0.6": 37, + "0.7": 17, + "0.8": 12, + "0.9": 7 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 3 + }, + "babel_mu1_abs": 0.3367546796798706 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8360996842384338, + "dcs_abs": 0.8378686904907227, + "frac_sign_flipped": 0.018880208333333332, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8351651430130005, + "0.6": 0.8351997137069702, + "0.7": 0.83649080991745, + "0.8": 0.8362331390380859, + "0.9": 0.8362571597099304 + }, + "unique_frac_signed": { + "0.5": 0.9689127604166666, + "0.6": 0.99072265625, + "0.7": 0.9977213541666666, + "0.8": 0.9990234375, + "0.9": 0.9991861979166666 + }, + "dedup_dcs_signed": { + "0.5": 0.8092021641205065, + "0.6": 0.8274512788630091, + "0.7": 0.8345847436188099, + "0.8": 0.8354165051132441, + "0.9": 0.8355766118911561 + }, + "matched_frac_signed": { + "0.5": 0.859375, + "0.6": 0.8486328125, + "0.7": 0.818359375, + "0.8": 0.7574869791666666, + "0.9": 0.6184895833333334 + }, + "matched_cos_abs": { + "0.5": 0.8372001647949219, + "0.6": 0.837662935256958, + "0.7": 0.8385027050971985, + "0.8": 0.8378106951713562, + "0.9": 0.8376309275627136 + }, + "unique_frac_abs": { + "0.5": 0.9596354166666666, + "0.6": 0.9830729166666666, + "0.7": 0.9920247395833334, + "0.8": 0.9947916666666666, + "0.9": 0.99658203125 + }, + "dedup_dcs_abs": { + "0.5": 0.8034069289763768, + "0.6": 0.8234837449466188, + "0.7": 0.8318154276639689, + "0.8": 0.833447097800672, + "0.9": 0.8347679312282708 + }, + "matched_frac_abs": { + "0.5": 0.8541666666666666, + "0.6": 0.8430989583333334, + "0.7": 0.8154296875, + "0.8": 0.7545572916666666, + "0.9": 0.6162109375 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8310664296150208, + "dcs_abs": 0.8334305882453918, + "frac_sign_flipped": 0.020833333333333332, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8281636834144592, + "0.6": 0.8314689993858337, + "0.7": 0.8307958841323853, + "0.8": 0.8309184908866882, + "0.9": 0.8309728503227234 + }, + "unique_frac_signed": { + "0.5": 0.9680989583333334, + "0.6": 0.9925130208333334, + "0.7": 0.9982096354166666, + "0.8": 0.9990234375, + "0.9": 0.9993489583333334 + }, + "dedup_dcs_signed": { + "0.5": 0.8017443992430344, + "0.6": 0.8252438083097029, + "0.7": 0.8293084566054555, + "0.8": 0.8301070470479317, + "0.9": 0.8304318523732945 + }, + "matched_frac_signed": { + "0.5": 0.8499348958333334, + "0.6": 0.84765625, + "0.7": 0.8095703125, + "0.8": 0.7464192708333334, + "0.9": 0.6087239583333334 + }, + "matched_cos_abs": { + "0.5": 0.8305121064186096, + "0.6": 0.8334792852401733, + "0.7": 0.8334130644798279, + "0.8": 0.8333910703659058, + "0.9": 0.833544909954071 + }, + "unique_frac_abs": { + "0.5": 0.9586588541666666, + "0.6": 0.9855143229166666, + "0.7": 0.9933268229166666, + "0.8": 0.9954427083333334, + "0.9": 0.9970703125 + }, + "dedup_dcs_abs": { + "0.5": 0.796177784310809, + "0.6": 0.8214057734585367, + "0.7": 0.8278515515169904, + "0.8": 0.8295930641858529, + "0.9": 0.83110288385069 + }, + "matched_frac_abs": { + "0.5": 0.8430989583333334, + "0.6": 0.8447265625, + "0.7": 0.8072916666666666, + "0.8": 0.7454427083333334, + "0.9": 0.6077473958333334 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8369827270507812, + "dcs_abs": 0.8387919068336487, + "frac_sign_flipped": 0.019205729166666668, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8329333662986755, + "0.6": 0.8375393152236938, + "0.7": 0.8371511697769165, + "0.8": 0.8370564579963684, + "0.9": 0.8370863795280457 + }, + "unique_frac_signed": { + "0.5": 0.97119140625, + "0.6": 0.9923502604166666, + "0.7": 0.9978841145833334, + "0.8": 0.9990234375, + "0.9": 0.9991861979166666 + }, + "dedup_dcs_signed": { + "0.5": 0.808937727328157, + "0.6": 0.8311323575714292, + "0.7": 0.8353798538252402, + "0.8": 0.8362390200491063, + "0.9": 0.8364051568884557 + }, + "matched_frac_signed": { + "0.5": 0.8619791666666666, + "0.6": 0.8561197916666666, + "0.7": 0.818359375, + "0.8": 0.7574869791666666, + "0.9": 0.6236979166666666 + }, + "matched_cos_abs": { + "0.5": 0.835595428943634, + "0.6": 0.8401258587837219, + "0.7": 0.8396346569061279, + "0.8": 0.839143693447113, + "0.9": 0.8390862941741943 + }, + "unique_frac_abs": { + "0.5": 0.9615885416666666, + "0.6": 0.9845377604166666, + "0.7": 0.9925130208333334, + "0.8": 0.9951171875, + "0.9": 0.9969075520833334 + }, + "dedup_dcs_abs": { + "0.5": 0.8034989899412418, + "0.6": 0.8271356314750543, + "0.7": 0.8333483297222605, + "0.8": 0.8350463121314533, + "0.9": 0.8364914635118719 + }, + "matched_frac_abs": { + "0.5": 0.8561197916666666, + "0.6": 0.8522135416666666, + "0.7": 0.8170572916666666, + "0.8": 0.7545572916666666, + "0.9": 0.6220703125 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 18, + "n_dense_std": 1.0, + "l0_mean": 32, + "l0_std": 0.10067001902668278, + "recon_r2_mean": 0.8960437740512998, + "recon_r2_std": null, + "l1_per_token_mean": 323.90742854567094, + "l1_per_token_std": 1.122885864644139, + "babel_mu1_signed_mean": 0.3215125898520152, + "babel_mu1_signed_std": 0.001176985979332288, + "n_dup_signed_per_threshold": { + "0.5": 182.0, + "0.6": 48.0, + "0.7": 11.333333333333334, + "0.8": 4.666666666666667, + "0.9": 3.3333333333333335 + }, + "n_dup_signed_band": 16.833333333333332, + "n_dup_signed_band_std": 3.556098048891978, + "n_dup_clusters_signed_per_threshold": { + "0.5": 88.0, + "0.6": 23.0, + "0.7": 5.0, + "0.8": 1.6666666666666667, + "0.9": 1.0 + }, + "n_dup_clusters_signed_band": 7.666666666666667, + "n_dup_clusters_signed_band_std": 1.6645820296198481, + "largest_dup_signed_per_threshold": { + "0.5": 3.3333333333333335, + "0.6": 3.3333333333333335, + "0.7": 3.3333333333333335, + "0.8": 3.3333333333333335, + "0.9": 3.3333333333333335 + }, + "largest_dup_signed_band": 3.3333333333333335, + "largest_dup_signed_band_std": 0.5773502691896258, + "dcs_signed_mean": 0.834716280301412, + "dcs_signed_std": 0.0031915511396219728, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.8066280968972327, + "0.6": 0.8279424815813804, + "0.7": 0.833091018016502, + "0.8": 0.8339208574034274, + "0.9": 0.8341378737176354 + }, + "dedup_dcs_signed_band": 0.8322730576797363, + "dedup_dcs_signed_band_std": 0.0031265814947999964, + "matched_cos_signed_per_threshold": { + "0.5": 0.8320873975753784, + "0.6": 0.8347360094388326, + "0.7": 0.8348126212755839, + "0.8": 0.8347360293070475, + "0.9": 0.8347721298535665 + }, + "matched_cos_signed_band": 0.8347641974687576, + "matched_cos_signed_band_std": 0.003278066986630508, + "unique_frac_signed_per_threshold": { + "0.5": 0.9694010416666666, + "0.6": 0.9918619791666666, + "0.7": 0.9979383680555555, + "0.8": 0.9990234375, + "0.9": 0.9992404513888888 + }, + "unique_frac_signed_band": 0.9970160590277778, + "unique_frac_signed_band_std": 0.0003160584906069042, + "matched_frac_signed_per_threshold": { + "0.5": 0.8570963541666666, + "0.6": 0.8508029513888888, + "0.7": 0.8154296875, + "0.8": 0.7537977430555555, + "0.9": 0.6169704861111112 + }, + "matched_frac_signed_band": 0.7592502170138888, + "matched_frac_signed_band_std": 0.005563891303106848, + "babel_mu1_abs_mean": 0.33806049823760986, + "babel_mu1_abs_std": 0.0012355827217323833, + "n_dup_abs_per_threshold": { + "0.5": 238.33333333333334, + "0.6": 93.0, + "0.7": 43.333333333333336, + "0.8": 28.666666666666668, + "0.9": 18.0 + }, + "n_dup_abs_band": 45.75, + "n_dup_abs_band_std": 7.1545440106270926, + "n_dup_clusters_abs_per_threshold": { + "0.5": 115.33333333333333, + "0.6": 45.0, + "0.7": 20.666666666666668, + "0.8": 13.666666666666666, + "0.9": 8.333333333333334 + }, + "n_dup_clusters_abs_band": 21.916666666666668, + "n_dup_clusters_abs_band_std": 3.412599204907212, + "largest_dup_abs_per_threshold": { + "0.5": 3.3333333333333335, + "0.6": 3.3333333333333335, + "0.7": 3.3333333333333335, + "0.8": 3.3333333333333335, + "0.9": 3.3333333333333335 + }, + "largest_dup_abs_band": 3.3333333333333335, + "largest_dup_abs_band_std": 0.5773502691896258, + "dcs_abs_mean": 0.8366970618565878, + "dcs_abs_std": 0.002866264030657456, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.8010279010761425, + "0.6": 0.8240083832934033, + "0.7": 0.8310051029677399, + "0.8": 0.8326954913726595, + "0.9": 0.8341207595302776 + }, + "dedup_dcs_abs_band": 0.8304574342910201, + "dedup_dcs_abs_band_std": 0.0027825610883345613, + "matched_cos_abs_per_threshold": { + "0.5": 0.8344359000523885, + "0.6": 0.8370893597602844, + "0.7": 0.8371834754943848, + "0.8": 0.8367818196614584, + "0.9": 0.836754043896993 + }, + "matched_cos_abs_band": 0.8369521747032802, + "matched_cos_abs_band_std": 0.0031302403283404627, + "unique_frac_abs_per_threshold": { + "0.5": 0.9599609375, + "0.6": 0.984375, + "0.7": 0.9926215277777778, + "0.8": 0.9951171875, + "0.9": 0.9968532986111112 + }, + "unique_frac_abs_band": 0.9922417534722222, + "unique_frac_abs_band_std": 0.0006108035074434534, + "matched_frac_abs_per_threshold": { + "0.5": 0.8511284722222222, + "0.6": 0.8466796875, + "0.7": 0.813259548611111, + "0.8": 0.7515190972222222, + "0.9": 0.6153428819444445 + }, + "matched_frac_abs_band": 0.7567003038194445, + "matched_frac_abs_band_std": 0.005114882588830758, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/config.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6b8b0bf9a79be3eaf8cb64f0d93fcb3200327e0f --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8d0984b271b418187d4b93a6ed7f9e551dbb1c4e --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ccb1d0f953d1f1b18288acbe293c7afba7f4108e7d473c4160f429251a7f176 +size 18890116 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..89adffb4342cfbb01633fb90b0962750391476b5 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f09b3238715cc90a8d93d8a6d414d3ecbafd760540dbb4393a7bc6fe75f6184d +size 25920 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7a6f60f5cc71019a4a0542cfefddfcc19fc1351 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd15cedfe7e20454239bdd2352cd0b12387bbb4e5dc221099889bf66b581cb65 +size 13607 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..46524d1c588b523412704e3d3117ccfa5833341e --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.990659716796873, + "l0_per_token_0.001": 31.990659716796873, + "l0_per_token_0.01": 31.990659716796873, + "l0_per_token_0.05": 31.990659716796873, + "l0_per_token_0.1": 31.990659716796873, + "l0_per_token_0.5": 31.990659716796873, + "l0_per_token_1.0": 31.990659716796873, + "l1_per_token": 324.9382562512766, + "r2_score": 0.8961075294995073, + "l2_loss": 2.436531290409442 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..253d88f41a48064d16fd9e33b1984bb5dc339911 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e87f101f86202300fa524d40992d5994de72c7a4f8ff1d2761f121f149f13444 +size 13518 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8039cb1c72a205b24182ea67de1dce7cb96226b9 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fa1e2e14fdcb43a60e2f503e0f7e15a8f84c6b050cc31ffadf03d02a2b6e771 +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b7cddc2997d33d484e46b618fc6de1bc0ea84e5 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a316ce9fb5de4a5b06d3ba334ec188595a4ee7d57d5258c7c13f09e737d1da41 +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/config.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..78eb67885d6081b1eee04627d0a79573862692b1 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..429f16f390249aedf2189ce886b41e45838c2206 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8e77623ca4f65969642c1c02d902d4b7bb75b78170948060d80142541f55668 +size 18890116 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b720fc9adb31c4dc6965045b2b2bfa3ab1173376 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f53b901984071a2f3a2cedc050f10e9285219be7d3e586418ff1ace6d7ad000d +size 25920 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d59c57b625d8eba15d199fb486309093122272b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:224099500682634a3c746e654f414d4cfb6df6c0492b2bff0cad4d0f7e80802c +size 13607 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..6bf24d401a68dff51d685cf3d7b86c6bc6de3a95 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.92978974609375, + "l0_per_token_0.001": 31.92978974609375, + "l0_per_token_0.01": 31.92978974609375, + "l0_per_token_0.05": 31.92978974609375, + "l0_per_token_0.1": 31.92978974609375, + "l0_per_token_0.5": 31.92978974609375, + "l0_per_token_1.0": 31.92978974609375, + "l1_per_token": 322.7108991769086, + "r2_score": 0.8961027107039853, + "l2_loss": 2.4366443028925913 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7287653b81481201ef6b31191889f1c8212360db --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba3fc447b14b8a2fcbffabd58a9e95685806a8a40f4b3a87b20fc241e2029624 +size 13518 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..85d60a8782c27912f78bcc8693fa31ae67c5909a --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea8d79122fab41c18e181fc0fede64c82a0fb419fd0d40d61bb9d84fe885546c +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a657820b027404d92b3f680405fd49d0a6713623 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cc1e3bcf985b31508c45014fe87ea9fff372070af5ba208c660dea952c865f8 +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/config.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..33db7efae94f3b46add2453de64aeae70e044c21 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d6201dd4328d50b5694d49757df5d0ef294fce41 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67029f3aebad34ad1a6623c744a53e57e4372d73723bf14c444ebc0e2315469b +size 18890116 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a12f5974d778c00e88131c914e28586cfd5bb8ef --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11cac3b5b9bf8353e92e774770126fa8f65586a87be7e049c6395f0d5f74f41c +size 25920 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..d95c8dd20fffbfc696d07f1f45482b9aefb3ab31 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9710ca14985db7196d95a49df0784ded90aa56b1c08ac2cd15a1fd8d604a8756 +size 13607 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..f2e39d23651e56649d61d4cebb4fd66a9d9133fa --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.7940185546875, + "l0_per_token_0.001": 31.7940185546875, + "l0_per_token_0.01": 31.7940185546875, + "l0_per_token_0.05": 31.7940185546875, + "l0_per_token_0.1": 31.7940185546875, + "l0_per_token_0.5": 31.7940185546875, + "l0_per_token_1.0": 31.7940185546875, + "l1_per_token": 324.0731302088276, + "r2_score": 0.8959210819504068, + "l2_loss": 2.4409039392184977 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b91e1d632302e11009dda2849492570fd69c970e --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f754cd1867f04d42cc909710056adbffccfa3d6646fa36d23ee47df640ea0801 +size 13518 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..bab171c8c3d81538a9bfc6d154a4e98de0b2d0bc --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d37cd4b7a460aa09fa92b19df89d270de2a314e112b41e3d6a00c9fa7be4daf6 +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e41c1212c33e36f701cd7f90f1459550f66dffcd --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:231c95f0b81931fd17d168ede5c970b4fd6c88ec2a4b195e210e277c09970fc8 +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/code_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..2482fbe53983e0dd31fc4901b2be313c25f03934 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 63.79612, + "recon_r2": 0.9188253071248624, + "l1_per_token": 447.86650615881683 + }, + "seed1": { + "l0": 63.83540833333333, + "recon_r2": 0.9189602152984583, + "l1_per_token": 447.98201194275896 + }, + "seed2": { + "l0": 63.611535, + "recon_r2": 0.9187433040530715, + "l1_per_token": 446.5362078690215 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7784013690103485, + "mcc_abs": 0.7784833503770058, + "act_iou": 0.4727742849532288, + "act_iou_n_pairs": 3072, + "input_r2": 0.9652325715166491, + "matched_mcc_signed": { + "0.6": 0.8829437756390087, + "0.7": 0.9005908796539822, + "0.8": 0.9200234726874946, + "0.9": 0.9434004356685523 + }, + "matched_mcc_abs": { + "0.6": 0.8804491133740322, + "0.7": 0.8989236800398284, + "0.8": 0.918959417457873, + "0.9": 0.9428448815148411 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7741073448112864, + "mcc_abs": 0.7741458444539396, + "act_iou": 0.4663208407057124, + "act_iou_n_pairs": 3072, + "input_r2": 0.9642315317884481, + "matched_mcc_signed": { + "0.6": 0.8774258749729107, + "0.7": 0.8980083775724788, + "0.8": 0.9180733356291833, + "0.9": 0.9428990206652855 + }, + "matched_mcc_abs": { + "0.6": 0.8748041785376705, + "0.7": 0.8965307207177623, + "0.8": 0.9173897882826051, + "0.9": 0.9428288741508023 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7808676201489536, + "mcc_abs": 0.7809224646477185, + "act_iou": 0.4742668232508754, + "act_iou_n_pairs": 3072, + "input_r2": 0.9649290968323283, + "matched_mcc_signed": { + "0.6": 0.8805660140179383, + "0.7": 0.8995498946061612, + "0.8": 0.9204881910611972, + "0.9": 0.9445341969139333 + }, + "matched_mcc_abs": { + "0.6": 0.8782923819595705, + "0.7": 0.898313176543463, + "0.8": 0.9194821789993973, + "0.9": 0.944302836845964 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 63.74768777777778, + "l0_std": 0.11953692925820504, + "recon_r2_mean": 0.9188429421587975, + "recon_r2_std": 0.0001095256495540178, + "l1_per_token_mean": 447.4615753235324, + "l1_per_token_std": 0.8034700311349855, + "input_r2_mean": 0.9647977333791419, + "input_r2_std": 0.0005132858871856676, + "mcc_signed_mean": 0.7777921113235294, + "mcc_signed_std": 0.0034210710102216394, + "mcc_abs_mean": 0.7778505531595546, + "mcc_abs_std": 0.0034323417008644973, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8803118882099525, + "0.7": 0.899383050610874, + "0.8": 0.9195283331259584, + "0.9": 0.943611217749257 + }, + "matched_mcc_signed_band": 0.9107086224240105, + "matched_mcc_abs_per_threshold": { + "0.6": 0.877848557957091, + "0.7": 0.8979225257670178, + "0.8": 0.9186104615799585, + "0.9": 0.9433255308372025 + }, + "matched_mcc_abs_band": 0.9094267690353174, + "act_iou_mean": 0.47112064963660555, + "act_iou_std": 0.0042232146490909705, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/dict_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..606af2977da444b66dd536532f1cd22c3e031e6d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 64, + "l0_raw": 63.85828901367187, + "recon_r2": 0.9187321334055357, + "l1_per_token": 447.866415478025, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 61, + "dense_rate": 0.019856770833333332, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 60, + "0.6": 12, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "n_dup_clusters_signed": { + "0.5": 30, + "0.6": 6, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_signed": 0.2571597099304199, + "n_dup_abs": { + "0.5": 239, + "0.6": 172, + "0.7": 145, + "0.8": 131, + "0.9": 98 + }, + "n_dup_clusters_abs": { + "0.5": 118, + "0.6": 85, + "0.7": 72, + "0.8": 65, + "0.9": 49 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 2 + }, + "babel_mu1_abs": 0.3093826472759247 + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 64, + "l0_raw": 63.87720483398437, + "recon_r2": 0.9188542715368677, + "l1_per_token": 447.8845773215997, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 65, + "dense_rate": 0.021158854166666668, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 65, + "0.6": 22, + "0.7": 4, + "0.8": 2, + "0.9": 2 + }, + "n_dup_clusters_signed": { + "0.5": 32, + "0.6": 11, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_signed": 0.2583339512348175, + "n_dup_abs": { + "0.5": 253, + "0.6": 194, + "0.7": 169, + "0.8": 148, + "0.9": 106 + }, + "n_dup_clusters_abs": { + "0.5": 125, + "0.6": 96, + "0.7": 84, + "0.8": 74, + "0.9": 53 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.31312984228134155 + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 64, + "l0_raw": 63.669830029296875, + "recon_r2": 0.9186550573813604, + "l1_per_token": 446.5086006494308, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 59, + "dense_rate": 0.019205729166666668, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 67, + "0.6": 18, + "0.7": 2, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 33, + "0.6": 9, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.25743788480758667, + "n_dup_abs": { + "0.5": 233, + "0.6": 169, + "0.7": 144, + "0.8": 128, + "0.9": 94 + }, + "n_dup_clusters_abs": { + "0.5": 115, + "0.6": 83, + "0.7": 72, + "0.8": 64, + "0.9": 47 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3075447976589203 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.7904099822044373, + "dcs_abs": 0.7923679351806641, + "frac_sign_flipped": 0.026041666666666668, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7900945544242859, + "0.6": 0.7911986112594604, + "0.7": 0.7906279563903809, + "0.8": 0.7906638979911804, + "0.9": 0.7906638979911804 + }, + "unique_frac_signed": { + "0.5": 0.98974609375, + "0.6": 0.9972330729166666, + "0.7": 0.99951171875, + "0.8": 0.9996744791666666, + "0.9": 0.9996744791666666 + }, + "dedup_dcs_signed": { + "0.5": 0.7819929989345837, + "0.6": 0.7890094223936709, + "0.7": 0.7902419075835496, + "0.8": 0.7904065204202197, + "0.9": 0.7904065204202197 + }, + "matched_frac_signed": { + "0.5": 0.8434244791666666, + "0.6": 0.8095703125, + "0.7": 0.7561848958333334, + "0.8": 0.6728515625, + "0.9": 0.4954427083333333 + }, + "matched_cos_abs": { + "0.5": 0.79851895570755, + "0.6": 0.7995524406433105, + "0.7": 0.7992861866950989, + "0.8": 0.7979529500007629, + "0.9": 0.796461284160614 + }, + "unique_frac_abs": { + "0.5": 0.95947265625, + "0.6": 0.9698893229166666, + "0.7": 0.9742838541666666, + "0.8": 0.9772135416666666, + "0.9": 0.9833984375 + }, + "dedup_dcs_abs": { + "0.5": 0.7661571034986991, + "0.6": 0.7754773752919087, + "0.7": 0.7787316265554788, + "0.8": 0.7797704283536101, + "0.9": 0.7832387823727913 + }, + "matched_frac_abs": { + "0.5": 0.8313802083333334, + "0.6": 0.8011067708333334, + "0.7": 0.7473958333333334, + "0.8": 0.6656901041666666, + "0.9": 0.4905598958333333 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.7866203188896179, + "dcs_abs": 0.7883960604667664, + "frac_sign_flipped": 0.023111979166666668, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7862421274185181, + "0.6": 0.7870213985443115, + "0.7": 0.7867511510848999, + "0.8": 0.7868775725364685, + "0.9": 0.7868775725364685 + }, + "unique_frac_signed": { + "0.5": 0.9895833333333334, + "0.6": 0.99755859375, + "0.7": 0.9996744791666666, + "0.8": 0.9998372395833334, + "0.9": 0.9998372395833334 + }, + "dedup_dcs_signed": { + "0.5": 0.7780521052579086, + "0.6": 0.7850999595830217, + "0.7": 0.7864950471945727, + "0.8": 0.7867495000148969, + "0.9": 0.7867495000148969 + }, + "matched_frac_signed": { + "0.5": 0.845703125, + "0.6": 0.8095703125, + "0.7": 0.7493489583333334, + "0.8": 0.6611328125, + "0.9": 0.48828125 + }, + "matched_cos_abs": { + "0.5": 0.7936978340148926, + "0.6": 0.7937197089195251, + "0.7": 0.7938206791877747, + "0.8": 0.7932776808738708, + "0.9": 0.7919778227806091 + }, + "unique_frac_abs": { + "0.5": 0.9611002604166666, + "0.6": 0.9718424479166666, + "0.7": 0.9763997395833334, + "0.8": 0.9788411458333334, + "0.9": 0.984375 + }, + "dedup_dcs_abs": { + "0.5": 0.7628231949638575, + "0.6": 0.7713705048760554, + "0.7": 0.775086304434808, + "0.8": 0.7764928341105891, + "0.9": 0.7796031692996621 + }, + "matched_frac_abs": { + "0.5": 0.8326822916666666, + "0.6": 0.7985026041666666, + "0.7": 0.7428385416666666, + "0.8": 0.6546223958333334, + "0.9": 0.4840494791666667 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.791556179523468, + "dcs_abs": 0.7928640842437744, + "frac_sign_flipped": 0.0263671875, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7901358008384705, + "0.6": 0.79096919298172, + "0.7": 0.791527271270752, + "0.8": 0.7918018698692322, + "0.9": 0.7918018698692322 + }, + "unique_frac_signed": { + "0.5": 0.9890950520833334, + "0.6": 0.9967447916666666, + "0.7": 0.99951171875, + "0.8": 0.9998372395833334, + "0.9": 0.9998372395833334 + }, + "dedup_dcs_signed": { + "0.5": 0.7815194110832333, + "0.6": 0.7883944234733159, + "0.7": 0.7911407833453268, + "0.8": 0.7916729958669748, + "0.9": 0.7916729958669748 + }, + "matched_frac_signed": { + "0.5": 0.849609375, + "0.6": 0.8121744791666666, + "0.7": 0.759765625, + "0.8": 0.6692708333333334, + "0.9": 0.4967447916666667 + }, + "matched_cos_abs": { + "0.5": 0.7986788749694824, + "0.6": 0.799353837966919, + "0.7": 0.8001173138618469, + "0.8": 0.799094557762146, + "0.9": 0.7974357604980469 + }, + "unique_frac_abs": { + "0.5": 0.9599609375, + "0.6": 0.9700520833333334, + "0.7": 0.9744466145833334, + "0.8": 0.9775390625, + "0.9": 0.9837239583333334 + }, + "dedup_dcs_abs": { + "0.5": 0.7667005215771496, + "0.6": 0.7754148558403056, + "0.7": 0.7796716077621871, + "0.8": 0.7811461448436603, + "0.9": 0.7844566628336906 + }, + "matched_frac_abs": { + "0.5": 0.833984375, + "0.6": 0.802734375, + "0.7": 0.7522786458333334, + "0.8": 0.6643880208333334, + "0.9": 0.4931640625 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 62, + "n_dense_std": 3.055050463303893, + "l0_mean": 64, + "l0_std": 0.11465811983815727, + "recon_r2_mean": 0.9187471541079213, + "recon_r2_std": null, + "l1_per_token_mean": 447.4198644830185, + "l1_per_token_std": 0.7892298590769873, + "babel_mu1_signed_mean": 0.25764384865760803, + "babel_mu1_signed_std": 0.0006136175444648327, + "n_dup_signed_per_threshold": { + "0.5": 64.0, + "0.6": 17.333333333333332, + "0.7": 2.6666666666666665, + "0.8": 1.3333333333333333, + "0.9": 1.3333333333333333 + }, + "n_dup_signed_band": 5.666666666666666, + "n_dup_signed_band_std": 1.607275126832159, + "n_dup_clusters_signed_per_threshold": { + "0.5": 31.666666666666668, + "0.6": 8.666666666666666, + "0.7": 1.3333333333333333, + "0.8": 0.6666666666666666, + "0.9": 0.6666666666666666 + }, + "n_dup_clusters_signed_band": 2.833333333333333, + "n_dup_clusters_signed_band_std": 0.8036375634160795, + "largest_dup_signed_per_threshold": { + "0.5": 2.6666666666666665, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 1.6666666666666667, + "0.9": 1.6666666666666667 + }, + "largest_dup_signed_band": 1.8333333333333335, + "largest_dup_signed_band_std": 0.2886751345948129, + "dcs_signed_mean": 0.7895288268725077, + "dcs_signed_std": 0.0025832162295167326, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7805215050919085, + "0.6": 0.7875012684833361, + "0.7": 0.789292579374483, + "0.8": 0.7896096721006972, + "0.9": 0.7896096721006972 + }, + "dedup_dcs_signed_band": 0.7890032980148034, + "dedup_dcs_signed_band_std": 0.002390150159310452, + "matched_cos_signed_per_threshold": { + "0.5": 0.7888241608937582, + "0.6": 0.7897297342618307, + "0.7": 0.7896354595820109, + "0.8": 0.7897811134656271, + "0.9": 0.7897811134656271 + }, + "matched_cos_signed_band": 0.789731855193774, + "matched_cos_signed_band_std": 0.0024954309467312024, + "unique_frac_signed_per_threshold": { + "0.5": 0.989474826388889, + "0.6": 0.9971788194444443, + "0.7": 0.9995659722222222, + "0.8": 0.9997829861111112, + "0.9": 0.9997829861111112 + }, + "unique_frac_signed_band": 0.9990776909722222, + "unique_frac_signed_band_std": 0.00013080038467061658, + "matched_frac_signed_per_threshold": { + "0.5": 0.8462456597222222, + "0.6": 0.8104383680555555, + "0.7": 0.755099826388889, + "0.8": 0.6677517361111112, + "0.9": 0.4934895833333333 + }, + "matched_frac_signed_band": 0.6816948784722222, + "matched_frac_signed_band_std": 0.004023453740893355, + "babel_mu1_abs_mean": 0.3100190957387288, + "babel_mu1_abs_std": 0.002846397871514716, + "n_dup_abs_per_threshold": { + "0.5": 241.66666666666666, + "0.6": 178.33333333333334, + "0.7": 152.66666666666666, + "0.8": 135.66666666666666, + "0.9": 99.33333333333333 + }, + "n_dup_abs_band": 141.5, + "n_dup_abs_band_std": 11.127106542133943, + "n_dup_clusters_abs_per_threshold": { + "0.5": 119.33333333333333, + "0.6": 88.0, + "0.7": 76.0, + "0.8": 67.66666666666667, + "0.9": 49.666666666666664 + }, + "n_dup_clusters_abs_band": 70.33333333333334, + "n_dup_clusters_abs_band_std": 5.592033023269206, + "largest_dup_abs_per_threshold": { + "0.5": 3.0, + "0.6": 3.0, + "0.7": 2.6666666666666665, + "0.8": 2.3333333333333335, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.5, + "largest_dup_abs_band_std": 0.25, + "dcs_abs_mean": 0.791209359963735, + "dcs_abs_std": 0.002448985814156362, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.7652269400132354, + "0.6": 0.7740875786694232, + "0.7": 0.7778298462508246, + "0.8": 0.7791364691026198, + "0.9": 0.782432871502048 + }, + "dedup_dcs_abs_band": 0.7783716913812289, + "dedup_dcs_abs_band_std": 0.002406703594546283, + "matched_cos_abs_per_threshold": { + "0.5": 0.796965221563975, + "0.6": 0.7975419958432516, + "0.7": 0.7977413932482401, + "0.8": 0.7967750628789266, + "0.9": 0.7952916224797567 + }, + "matched_cos_abs_band": 0.7968375186125438, + "matched_cos_abs_band_std": 0.0031697484875529217, + "unique_frac_abs_per_threshold": { + "0.5": 0.9601779513888888, + "0.6": 0.9705946180555555, + "0.7": 0.9750434027777778, + "0.8": 0.9778645833333334, + "0.9": 0.9838324652777778 + }, + "unique_frac_abs_band": 0.976833767361111, + "unique_frac_abs_band_std": 0.000901020164943702, + "matched_frac_abs_per_threshold": { + "0.5": 0.8326822916666666, + "0.6": 0.80078125, + "0.7": 0.7475043402777778, + "0.8": 0.6615668402777778, + "0.9": 0.4892578125 + }, + "matched_frac_abs_band": 0.6747775607638888, + "matched_frac_abs_band_std": 0.004248431412858567, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/config.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8661b7883db63a1f6d2922a16b302ff8a496f0ce --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..68cd1e6a2488e6149e56bfdf7f1edf59d8bfd33d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4316036b2542d71331aecd89ddcfb0d4743d99346616e16630998436918c48de +size 18890116 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a3ed596dea551d8cc8007e3bc429c1e4c93a758 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a5fea4086b2268977253deff168bd5b199680593bfbda58b605e0998b2c8a37 +size 25920 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7aa8f80b2c3e60295ab8ec3863c95ee3f83b25d9 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5795dc2946e1fc7248ae5b6bd51362c2ee50e0d2b0d4bbdb000a0e249c92f60 +size 13607 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e8cf5b65567c058a100bb39e303f48dadbb0165b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.85828901367187, + "l0_per_token_0.001": 63.85828901367187, + "l0_per_token_0.01": 63.85828901367187, + "l0_per_token_0.05": 63.85828901367187, + "l0_per_token_0.1": 63.85828901367187, + "l0_per_token_0.5": 63.85828901367187, + "l0_per_token_1.0": 63.85828901367187, + "l1_per_token": 447.866415478025, + "r2_score": 0.9187321334055357, + "l2_loss": 1.9059292642510934 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f369d019f5455b2786d939ef7cedb8850d080ac --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ee22b3a244cf427d61decb7c366961720b9c1dc34153ef03d696c9b89fbc52c +size 13518 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5cb839dc9da9c81c93214181fc44b4ef4368ed42 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dff520c21f14e48c33fa24db294096153f58b1d2150f35497ad4582c19bf2d14 +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..31b3fb92a7a94f3f430dece0b669e1f8f34e643f --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c24ddb02b31ea407184290248e7de7507d007210114673b66bb26ef7e2eaf243 +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/config.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce45a08b6913db69f2d2155ef8d8951ada06ad6 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a1ffa1ccb43382215877d710a2d197e69a7a8548 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c23e557bf9c79a35abd1c795d4bbf528cae0cfd062536b68afd43aa4599dd90b +size 18890116 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b0f832863516f27cee388bdfc430f11080560f9 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cd51941d73687e1bdf07c49c93e613c450874733dbd3d2512cb3b40b9154958 +size 25920 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2f178190834b983eedbb9c0d06923c2fb664db5 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52de4c4b37b7a5f78c8a9569036e628786ab9ae4e2f4c48e8c7c36013cb956ee +size 13607 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..2298327ef5c1555476cf23c0cadb595fe9b5b08b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.87720483398437, + "l0_per_token_0.001": 63.87720483398437, + "l0_per_token_0.01": 63.87720483398437, + "l0_per_token_0.05": 63.87720483398437, + "l0_per_token_0.1": 63.87720483398437, + "l0_per_token_0.5": 63.87720483398437, + "l0_per_token_1.0": 63.87720483398437, + "l1_per_token": 447.8845773215997, + "r2_score": 0.9188542715368677, + "l2_loss": 1.9030648278072504 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0c108f267317c13e366e320abd9eeedc59ddb2d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0063264cfa2fb81e0fa6371104531636a1c87a2390cc1020a8cee9492b58ee50 +size 13518 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b4d340a79ec601ee3d5f6090b29c62fd95f025e --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0663bdda55d45a8e2f6340321279548a7f4d2129b33646a1da0c101e200e291b +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..fce17dcf973aa377275a754bb41f2851b7d991c3 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5072c25e1b292f9e91b58564c65a337bfdf5cba5ca5a539b60ed950d9a203f5a +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/config.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..975d6bca46c0174ff9e6312f6bc2a9da92822fa8 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4ab1ac74dbe39feef219e840c25d8fe17a74d2b3 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b557c9cac55e0b6be73f327c6afc917246538a42817cb3408e537921b327aea +size 18890116 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f371cf9563273b7ff211c962292b53cb0fcf948 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aded8da90dccf8251ea47964d1fd562a452f67f2469d95a1c0445b040fa3081 +size 25920 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e7d80a26e9f0592266eb10dccddf3224fd94f6b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d612042bd0ea598534c9bdd89655b951e8d189858a67c0df8cafd71decbc618f +size 13607 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e53d839831e82326778f3110158b1f2aa30429cf --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.669830029296875, + "l0_per_token_0.001": 63.669830029296875, + "l0_per_token_0.01": 63.669830029296875, + "l0_per_token_0.05": 63.669830029296875, + "l0_per_token_0.1": 63.669830029296875, + "l0_per_token_0.5": 63.669830029296875, + "l0_per_token_1.0": 63.669830029296875, + "l1_per_token": 446.5086006494308, + "r2_score": 0.9186550573813604, + "l2_loss": 1.907736884608361 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..00e62dfd1f0d503b01f7ba2425c0e34746ba7269 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e61353d9aae6eefd453136f27d7b5a3371908fa385d30b215ae409bc8a0c7b4 +size 13518 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..56da2a008b05e4014a65785009e8e8d97dbdbd0e --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c35b4980f3a8944fff71a3a00ab77c442665ee6d03a068553bafa37bca4f66ea +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..864bca88965e02732a1d0fc441e8961054e21e1e --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59de9369108c06b9ec3c24f6914cbaa61dea54e0000d4d31bba3eee6f80c599d +size 13523 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/code_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..093b520ec1a152b5f8d2589c91520ecec4f973df --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 127.86067333333334, + "recon_r2": 0.9493663306785357, + "l1_per_token": 628.1545574319452 + }, + "seed1": { + "l0": 128.06305166666667, + "recon_r2": 0.9493195889591168, + "l1_per_token": 633.2669717607228 + }, + "seed2": { + "l0": 127.48401333333334, + "recon_r2": 0.9492988873775299, + "l1_per_token": 628.1836463517214 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7325845842487011, + "mcc_abs": 0.7326456840447038, + "act_iou": 0.40138830030755684, + "act_iou_n_pairs": 6144, + "input_r2": 0.9685735421882316, + "matched_mcc_signed": { + "0.6": 0.872061997200405, + "0.7": 0.8968841006142624, + "0.8": 0.9220399881295671, + "0.9": 0.9490878070546563 + }, + "matched_mcc_abs": { + "0.6": 0.8714843923137815, + "0.7": 0.8971540784114256, + "0.8": 0.9218512290856677, + "0.9": 0.9490543117414852 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7340728385865329, + "mcc_abs": 0.7341344452354949, + "act_iou": 0.40290263066737353, + "act_iou_n_pairs": 6144, + "input_r2": 0.9685279553224175, + "matched_mcc_signed": { + "0.6": 0.8707698373089605, + "0.7": 0.896272256084102, + "0.8": 0.9223054001883886, + "0.9": 0.948936325277871 + }, + "matched_mcc_abs": { + "0.6": 0.8704184952275635, + "0.7": 0.895828242983946, + "0.8": 0.9221456806061666, + "0.9": 0.9486458468742645 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7318772959787504, + "mcc_abs": 0.7319962592474982, + "act_iou": 0.39883858427231605, + "act_iou_n_pairs": 6144, + "input_r2": 0.9682864602718758, + "matched_mcc_signed": { + "0.6": 0.8681125416495169, + "0.7": 0.895718743189517, + "0.8": 0.9220150715114781, + "0.9": 0.9511806744091104 + }, + "matched_mcc_abs": { + "0.6": 0.866920362694018, + "0.7": 0.8955252138520895, + "0.8": 0.9218104249849631, + "0.9": 0.950992625346225 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 127.80257944444445, + "l0_std": 0.2938579976996306, + "recon_r2_mean": 0.9493282690050608, + "recon_r2_std": 3.4549343131713964e-05, + "l1_per_token_mean": 629.8683918481298, + "l1_per_token_std": 2.943292477485646, + "input_r2_mean": 0.968462652594175, + "input_r2_std": 0.0001542800745772516, + "mcc_signed_mean": 0.7328449062713281, + "mcc_signed_std": 0.0011206817131753414, + "mcc_abs_mean": 0.7329254628425655, + "mcc_abs_std": 0.0010962057113589885, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8703147920529607, + "0.7": 0.8962916999626271, + "0.8": 0.9221201532764779, + "0.9": 0.9497349355805459 + }, + "matched_mcc_signed_band": 0.9096153952181528, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8696077500784543, + "0.7": 0.8961691784158203, + "0.8": 0.9219357782255991, + "0.9": 0.9495642613206582 + }, + "matched_mcc_abs_band": 0.9093192420101329, + "act_iou_mean": 0.40104317174908216, + "act_iou_std": 0.0020538874279524264, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/dict_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..f17491e0a1fc8ba2f35d0bc02cdf1e4a4838da2d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 128, + "l0_raw": 127.98055083007813, + "recon_r2": 0.9492975474197132, + "l1_per_token": 628.1181409570913, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 254, + "dense_rate": 0.041341145833333336, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 123, + "0.6": 22, + "0.7": 4, + "0.8": 2, + "0.9": 2 + }, + "n_dup_clusters_signed": { + "0.5": 61, + "0.6": 11, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_signed": 0.2695484459400177, + "n_dup_abs": { + "0.5": 784, + "0.6": 656, + "0.7": 616, + "0.8": 590, + "0.9": 560 + }, + "n_dup_clusters_abs": { + "0.5": 391, + "0.6": 328, + "0.7": 308, + "0.8": 295, + "0.9": 280 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3618020713329315 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 128, + "l0_raw": 128.167773828125, + "recon_r2": 0.9492477488275837, + "l1_per_token": 633.1440717921007, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 267, + "dense_rate": 0.04345703125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 122, + "0.6": 20, + "0.7": 4, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 61, + "0.6": 10, + "0.7": 2, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.2700423002243042, + "n_dup_abs": { + "0.5": 770, + "0.6": 639, + "0.7": 589, + "0.8": 568, + "0.9": 534 + }, + "n_dup_clusters_abs": { + "0.5": 384, + "0.6": 319, + "0.7": 294, + "0.8": 284, + "0.9": 267 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3602128028869629 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 128, + "l0_raw": 127.5864357421875, + "recon_r2": 0.9492105655976164, + "l1_per_token": 628.0862270965011, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 262, + "dense_rate": 0.042643229166666664, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 118, + "0.6": 28, + "0.7": 4, + "0.8": 2, + "0.9": 2 + }, + "n_dup_clusters_signed": { + "0.5": 58, + "0.6": 14, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_signed": 0.26826897263526917, + "n_dup_abs": { + "0.5": 794, + "0.6": 667, + "0.7": 614, + "0.8": 594, + "0.9": 542 + }, + "n_dup_clusters_abs": { + "0.5": 396, + "0.6": 333, + "0.7": 307, + "0.8": 297, + "0.9": 271 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.35997581481933594 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.7187147736549377, + "dcs_abs": 0.7202315926551819, + "frac_sign_flipped": 0.030924479166666668, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.7163064479827881, + "0.6": 0.7186266183853149, + "0.7": 0.718694806098938, + "0.8": 0.7188082337379456, + "0.9": 0.7188082337379456 + }, + "unique_frac_signed": { + "0.5": 0.989990234375, + "0.6": 0.998291015625, + "0.7": 0.9996744791666666, + "0.8": 0.9999186197916666, + "0.9": 0.9999186197916666 + }, + "dedup_dcs_signed": { + "0.5": 0.7091363883228041, + "0.6": 0.7173984967230354, + "0.7": 0.7184608559667442, + "0.8": 0.7187497369741322, + "0.9": 0.7187497369741322 + }, + "matched_frac_signed": { + "0.5": 0.7825520833333334, + "0.6": 0.7265625, + "0.7": 0.64794921875, + "0.8": 0.5315755208333334, + "0.9": 0.2884114583333333 + }, + "matched_cos_abs": { + "0.5": 0.7291340827941895, + "0.6": 0.7310380339622498, + "0.7": 0.7311054468154907, + "0.8": 0.7307540774345398, + "0.9": 0.7301939725875854 + }, + "unique_frac_abs": { + "0.5": 0.9366048177083334, + "0.6": 0.947265625, + "0.7": 0.950927734375, + "0.8": 0.952880859375, + "0.9": 0.9554850260416666 + }, + "dedup_dcs_abs": { + "0.5": 0.6829104947003847, + "0.6": 0.6924872001400217, + "0.7": 0.6952284461294767, + "0.8": 0.6963215732976096, + "0.9": 0.697689406913317 + }, + "matched_frac_abs": { + "0.5": 0.7591145833333334, + "0.6": 0.7073567708333334, + "0.7": 0.6341145833333334, + "0.8": 0.5218098958333334, + "0.9": 0.2823893229166667 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.7212190628051758, + "dcs_abs": 0.723054826259613, + "frac_sign_flipped": 0.030924479166666668, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.7190249562263489, + "0.6": 0.7209453582763672, + "0.7": 0.7210639715194702, + "0.8": 0.7211802005767822, + "0.9": 0.7211802005767822 + }, + "unique_frac_signed": { + "0.5": 0.9900716145833334, + "0.6": 0.9979654947916666, + "0.7": 0.9996744791666666, + "0.8": 0.9998372395833334, + "0.9": 0.9998372395833334 + }, + "dedup_dcs_signed": { + "0.5": 0.7118861993367318, + "0.6": 0.7194785911900301, + "0.7": 0.7208292501745746, + "0.8": 0.7210628209868446, + "0.9": 0.7210628209868446 + }, + "matched_frac_signed": { + "0.5": 0.7890625, + "0.6": 0.7342122395833334, + "0.7": 0.6529947916666666, + "0.8": 0.53564453125, + "0.9": 0.294921875 + }, + "matched_cos_abs": { + "0.5": 0.7333496809005737, + "0.6": 0.7346113324165344, + "0.7": 0.7342210412025452, + "0.8": 0.7340626120567322, + "0.9": 0.7334355711936951 + }, + "unique_frac_abs": { + "0.5": 0.9356282552083334, + "0.6": 0.9461263020833334, + "0.7": 0.949951171875, + "0.8": 0.9518229166666666, + "0.9": 0.9551595052083334 + }, + "dedup_dcs_abs": { + "0.5": 0.6861426823985919, + "0.6": 0.6950351034077661, + "0.7": 0.6974741385056404, + "0.8": 0.6986976164237906, + "0.9": 0.7005479572835611 + }, + "matched_frac_abs": { + "0.5": 0.76708984375, + "0.6": 0.71728515625, + "0.7": 0.6414388020833334, + "0.8": 0.5266927083333334, + "0.9": 0.28955078125 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.716540515422821, + "dcs_abs": 0.7183859944343567, + "frac_sign_flipped": 0.030110677083333332, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.7140704393386841, + "0.6": 0.7162034511566162, + "0.7": 0.7164440751075745, + "0.8": 0.7166333794593811, + "0.9": 0.7166333794593811 + }, + "unique_frac_signed": { + "0.5": 0.9901529947916666, + "0.6": 0.998046875, + "0.7": 0.9996744791666666, + "0.8": 0.9999186197916666, + "0.9": 0.9999186197916666 + }, + "dedup_dcs_signed": { + "0.5": 0.7070389840033992, + "0.6": 0.714804616291076, + "0.7": 0.7162108576352086, + "0.8": 0.716575059685662, + "0.9": 0.716575059685662 + }, + "matched_frac_signed": { + "0.5": 0.7840169270833334, + "0.6": 0.7272135416666666, + "0.7": 0.6414388020833334, + "0.8": 0.5237630208333334, + "0.9": 0.2752278645833333 + }, + "matched_cos_abs": { + "0.5": 0.7277702689170837, + "0.6": 0.7290672659873962, + "0.7": 0.729396641254425, + "0.8": 0.7292221784591675, + "0.9": 0.7279932498931885 + }, + "unique_frac_abs": { + "0.5": 0.9361979166666666, + "0.6": 0.94677734375, + "0.7": 0.9510091145833334, + "0.8": 0.9527180989583334, + "0.9": 0.9562174479166666 + }, + "dedup_dcs_abs": { + "0.5": 0.6813370095721135, + "0.6": 0.6902643695066217, + "0.7": 0.6936628539794281, + "0.8": 0.6947431675798725, + "0.9": 0.6961198475134248 + }, + "matched_frac_abs": { + "0.5": 0.7586263020833334, + "0.6": 0.7096354166666666, + "0.7": 0.6292317708333334, + "0.8": 0.51416015625, + "0.9": 0.2696940104166667 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 261, + "n_dense_std": 6.557438524302, + "l0_mean": 128, + "l0_std": 0.2967415046978813, + "recon_r2_mean": 0.9492519539483043, + "recon_r2_std": null, + "l1_per_token_mean": 629.7828132818977, + "l1_per_token_std": 2.9109790024988746, + "babel_mu1_signed_mean": 0.269286572933197, + "babel_mu1_signed_std": 0.0009152080573750316, + "n_dup_signed_per_threshold": { + "0.5": 121.0, + "0.6": 23.333333333333332, + "0.7": 4.0, + "0.8": 1.3333333333333333, + "0.9": 1.3333333333333333 + }, + "n_dup_signed_band": 7.499999999999999, + "n_dup_signed_band_std": 1.5, + "n_dup_clusters_signed_per_threshold": { + "0.5": 60.0, + "0.6": 11.666666666666666, + "0.7": 2.0, + "0.8": 0.6666666666666666, + "0.9": 0.6666666666666666 + }, + "n_dup_clusters_signed_band": 3.7499999999999996, + "n_dup_clusters_signed_band_std": 0.75, + "largest_dup_signed_per_threshold": { + "0.5": 2.6666666666666665, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 1.6666666666666667, + "0.9": 1.6666666666666667 + }, + "largest_dup_signed_band": 1.8333333333333335, + "largest_dup_signed_band_std": 0.2886751345948129, + "dcs_signed_mean": 0.7188247839609782, + "dcs_signed_std": 0.0023412129554593966, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7093538572209783, + "0.6": 0.7172272347347137, + "0.7": 0.7185003212588424, + "0.8": 0.7187958725488796, + "0.9": 0.7187958725488796 + }, + "dedup_dcs_signed_band": 0.7183298252728287, + "dedup_dcs_signed_band_std": 0.0022835022900179066, + "matched_cos_signed_per_threshold": { + "0.5": 0.716467281182607, + "0.6": 0.7185918092727661, + "0.7": 0.7187342842419943, + "0.8": 0.718873937924703, + "0.9": 0.718873937924703 + }, + "matched_cos_signed_band": 0.7187684923410417, + "matched_cos_signed_band_std": 0.0023071188391110635, + "unique_frac_signed_per_threshold": { + "0.5": 0.9900716145833334, + "0.6": 0.9981011284722222, + "0.7": 0.9996744791666666, + "0.8": 0.9998914930555555, + "0.9": 0.9998914930555555 + }, + "unique_frac_signed_band": 0.9993896484374999, + "unique_frac_signed_band_std": 6.103515624994449e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.7852105034722223, + "0.6": 0.7293294270833334, + "0.7": 0.6474609375, + "0.8": 0.5303276909722223, + "0.9": 0.28618706597222215 + }, + "matched_frac_signed_band": 0.5483262803819445, + "matched_frac_signed_band_std": 0.006271602244108599, + "babel_mu1_abs_mean": 0.3606635630130768, + "babel_mu1_abs_std": 0.0009930718663990186, + "n_dup_abs_per_threshold": { + "0.5": 782.6666666666666, + "0.6": 654.0, + "0.7": 606.3333333333334, + "0.8": 584.0, + "0.9": 545.3333333333334 + }, + "n_dup_abs_band": 597.4166666666667, + "n_dup_abs_band_std": 12.93332259449726, + "n_dup_clusters_abs_per_threshold": { + "0.5": 390.3333333333333, + "0.6": 326.6666666666667, + "0.7": 303.0, + "0.8": 292.0, + "0.9": 272.6666666666667 + }, + "n_dup_clusters_abs_band": 298.58333333333337, + "n_dup_clusters_abs_band_std": 6.578056957288629, + "largest_dup_abs_per_threshold": { + "0.5": 3.0, + "0.6": 2.6666666666666665, + "0.7": 2.3333333333333335, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.25, + "largest_dup_abs_band_std": 0.25, + "dcs_abs_mean": 0.7205574711163839, + "dcs_abs_std": 0.002351413454018276, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.68346339555703, + "0.6": 0.6925955576848032, + "0.7": 0.6954551462048485, + "0.8": 0.6965874524337575, + "0.9": 0.698119070570101 + }, + "dedup_dcs_abs_band": 0.6956893067233776, + "dedup_dcs_abs_band_std": 0.0021322790251436918, + "matched_cos_abs_per_threshold": { + "0.5": 0.7300846775372823, + "0.6": 0.7315722107887268, + "0.7": 0.7315743764241537, + "0.8": 0.7313462893168131, + "0.9": 0.730540931224823 + }, + "matched_cos_abs_band": 0.7312584519386292, + "matched_cos_abs_band_std": 0.002615429755163125, + "unique_frac_abs_per_threshold": { + "0.5": 0.9361436631944445, + "0.6": 0.9467230902777778, + "0.7": 0.9506293402777778, + "0.8": 0.9524739583333334, + "0.9": 0.9556206597222222 + }, + "unique_frac_abs_band": 0.9513617621527778, + "unique_frac_abs_band_std": 0.0005172340214211049, + "matched_frac_abs_per_threshold": { + "0.5": 0.7616102430555557, + "0.6": 0.71142578125, + "0.7": 0.6349283854166666, + "0.8": 0.5208875868055556, + "0.9": 0.28054470486111116 + }, + "matched_frac_abs_band": 0.5369466145833333, + "matched_frac_abs_band_std": 0.006546808890386056, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/config.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..800c0ac8a2bdf4ea9d19bd42bae6c63a2420466c --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..92a3b765992e1efc7d2bc807f3e18b3fd4641600 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aea2e7032dece4db720fc155be5e6d026a122e5c2a925f24987939f6da9e8f98 +size 37776772 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8855ce18333eec8762fee8ad8504623b23666786 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:000a9dac7093183203067630640263faf7afb58ebd44c762cd02902d8b3d377e +size 50496 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..489cbf40d534a5ae6552f20651b395cb7da9d7e4 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98b8db73d6bd6515cef099d1fe602624c9fa5b9c8aac58abb417fe11ed3df49f +size 25895 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..cfe61a1773b93c1d0e7f8d244c8db24b0dede8c6 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.98055083007813, + "l0_per_token_0.001": 127.98055083007813, + "l0_per_token_0.01": 127.98055083007813, + "l0_per_token_0.05": 127.98055083007813, + "l0_per_token_0.1": 127.98055083007813, + "l0_per_token_0.5": 127.98055083007813, + "l0_per_token_1.0": 127.98055083007813, + "l1_per_token": 628.1181409570913, + "r2_score": 0.9492975474197132, + "l2_loss": 1.1890959144318736 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..12c326c453c190565b5e8315e406d7af62f6e680 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17b270aec66adf8c6f3bd0a0c0374c6d2824bbfd4be54fd019d3b0a7750c8df2 +size 25806 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fbe08907c254cc9b8b6defa6ea0a89129b95561 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8b18eb54f2f94a55ff42f429a15b85b94549902278c192d79ae65bba41f2550 +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..89c8c51391d07c9107f0108cedad74fcbf1bf3c2 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:114024b580e5dc4f435bca67aeb1a280fc1ad9e79326f62a332dc30502e9a1ed +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/config.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b9fbd7a21b669bb2b7076cc96b17e6cfe86e4d69 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1f5e61f81d549ac90731e6c4644200c85130bc0a --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2946cb61692077a3100fb9606059bad0aa74afbede4237007be95d169bde84e9 +size 37776772 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..deb29154d271352324779b2464ed91b375ac6898 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b1726da5f1f032cede2981b035778daa6d3e48500e98a02bb9878b480abf03f +size 50496 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b7275077daa3c22497e9df9e52259e6c6388250 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f412363464b40dce11cc449dde60f002b3ad8fc6c6ffd97a3e427d0574dd2078 +size 25895 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..9ca751e79782331ef9694062cbdaa08b095c6948 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.167773828125, + "l0_per_token_0.001": 128.167773828125, + "l0_per_token_0.01": 128.167773828125, + "l0_per_token_0.05": 128.167773828125, + "l0_per_token_0.1": 128.167773828125, + "l0_per_token_0.5": 128.167773828125, + "l0_per_token_1.0": 128.167773828125, + "l1_per_token": 633.1440717921007, + "r2_score": 0.9492477488275837, + "l2_loss": 1.1902638126188871 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..17faec2e904354cd80fb366e7a37ecfbfc982a0b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a996646a3c2cb4df94b13d4f4261aa33f9a099e9ab13efbbde475ab14cec813c +size 25806 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed9e3fb5c8d7b76acd2a0af3467651a8b3648172 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:985c85acb6e14a27fd67cf184c7a3e5fa88f65827e32957fa7e8abde39dc3bce +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f11764afe58f7bb5bf971b511cf465a4e302d17 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff7a543b8563da57a8fb889f9f9837becc4729e5116d9aab54447d83a1f88b18 +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/config.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..08c771d30493c2f267b68028023ab28a4e62bd16 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5fb8871d7de2e418b962bf912b652ab28737709d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a51635c4fe6245449afc2702f757542736a710a09ec7c5b72a7a57c4bad45d0 +size 37776772 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..441c5a27c8387f41f82a36e9c55df8754102c8d3 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb44757f57cd53b5e9eacefa27544e98d2955ab37a482778c2bed88db023103b +size 50496 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..681359438e6200dd419ba78b8b4971e99e2c513d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b516d6c8be8604f5897e8e524cdd0caa80e9ea71334896da55011690115e4636 +size 25895 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..7c7db17e62c2586ca0084e71355308b8dbd4eaae --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.5864357421875, + "l0_per_token_0.001": 127.5864357421875, + "l0_per_token_0.01": 127.5864357421875, + "l0_per_token_0.05": 127.5864357421875, + "l0_per_token_0.1": 127.5864357421875, + "l0_per_token_0.5": 127.5864357421875, + "l0_per_token_1.0": 127.5864357421875, + "l1_per_token": 628.0862270965011, + "r2_score": 0.9492105655976164, + "l2_loss": 1.191135849859481 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..033d271a5cd31e7a9e86c5ed00b6e4e1a31b82bd --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33fc833a9e8ddeaeb2d3480ab414c1e97d1ecc6a4b43587c8aaa3a1b45eb4c40 +size 25806 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..93b676cd7fda947792fcc769c621707bcfc2bcd8 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b6d16beaf837f7592dc28ba00d394be5f9183bd0650d5f1590529291c451f9c +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..078c12a4d792889f8d35206909501e3da3dae9e0 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e96a29c3daff3a80fde15996e3bd0d7a8d82a66c818934d0af3bd4040692820 +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/code_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..0494ec33937ce4d70d9ab0bca74a377c9358fcac --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 31.936433333333333, + "recon_r2": 0.9093115802389674, + "l1_per_token": 316.37819721142847 + }, + "seed1": { + "l0": 31.861498333333333, + "recon_r2": 0.9091946312179833, + "l1_per_token": 316.7402400649734 + }, + "seed2": { + "l0": 31.780518333333333, + "recon_r2": 0.909082575843282, + "l1_per_token": 311.80467461940447 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7709850736416103, + "mcc_abs": 0.7710095733855229, + "act_iou": 0.47058655669461297, + "act_iou_n_pairs": 6144, + "input_r2": 0.9685400561869463, + "matched_mcc_signed": { + "0.6": 0.864891565924189, + "0.7": 0.8870693077231048, + "0.8": 0.908359426878631, + "0.9": 0.9328987616772265 + }, + "matched_mcc_abs": { + "0.6": 0.8641079499983846, + "0.7": 0.8868627428842006, + "0.8": 0.9081524907540773, + "0.9": 0.9327193220740673 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7647152829048608, + "mcc_abs": 0.7648234082866301, + "act_iou": 0.46588982613067836, + "act_iou_n_pairs": 6144, + "input_r2": 0.9675255915975727, + "matched_mcc_signed": { + "0.6": 0.8622057068638199, + "0.7": 0.8855324829218776, + "0.8": 0.9085145660537229, + "0.9": 0.9328533056652091 + }, + "matched_mcc_abs": { + "0.6": 0.8609890795912634, + "0.7": 0.8846651353083114, + "0.8": 0.9077945875633325, + "0.9": 0.9329388672094673 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7658277175233499, + "mcc_abs": 0.7659835274461537, + "act_iou": 0.46448003379075303, + "act_iou_n_pairs": 6144, + "input_r2": 0.9673362956722467, + "matched_mcc_signed": { + "0.6": 0.8609167403742939, + "0.7": 0.8835015404122855, + "0.8": 0.9068626909182012, + "0.9": 0.9334266884195094 + }, + "matched_mcc_abs": { + "0.6": 0.8591809004081347, + "0.7": 0.8828903964325943, + "0.8": 0.9067203861230233, + "0.9": 0.9334746563474972 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 31.859483333333333, + "l0_std": 0.07797702850840124, + "recon_r2_mean": 0.909196262433411, + "recon_r2_std": 0.00011451091196317484, + "l1_per_token_mean": 314.97437063193547, + "l1_per_token_std": 2.7509995212362126, + "input_r2_mean": 0.9678006478189219, + "input_r2_std": 0.0006473034743183778, + "mcc_signed_mean": 0.7671760246899403, + "mcc_signed_std": 0.00334529791382872, + "mcc_abs_mean": 0.7672721697061022, + "mcc_abs_std": 0.003288253155291387, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8626713377207675, + "0.7": 0.8853677770190892, + "0.8": 0.907912227950185, + "0.9": 0.9330595852539817 + }, + "matched_mcc_signed_band": 0.8972527319860059, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8614259766659275, + "0.7": 0.8848060915417021, + "0.8": 0.9075558214801444, + "0.9": 0.9330442818770105 + }, + "matched_mcc_abs_band": 0.8967080428911961, + "act_iou_mean": 0.46698547220534814, + "act_iou_std": 0.003197301320574263, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/dict_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..7af27d49a01ae52cfe8371129f2f54b62d14256d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 32, + "l0_raw": 31.9758677734375, + "recon_r2": 0.9091598383835207, + "l1_per_token": 316.2830950868989, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 19, + "dense_rate": 0.0030924479166666665, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 637, + "0.6": 194, + "0.7": 69, + "0.8": 29, + "0.9": 24 + }, + "n_dup_clusters_signed": { + "0.5": 296, + "0.6": 82, + "0.7": 21, + "0.8": 3, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 25, + "0.6": 25, + "0.7": 25, + "0.8": 25, + "0.9": 24 + }, + "babel_mu1_signed": 0.3722122609615326, + "n_dup_abs": { + "0.5": 763, + "0.6": 284, + "0.7": 146, + "0.8": 94, + "0.9": 68 + }, + "n_dup_clusters_abs": { + "0.5": 356, + "0.6": 125, + "0.7": 59, + "0.8": 35, + "0.9": 23 + }, + "largest_dup_abs": { + "0.5": 25, + "0.6": 25, + "0.7": 25, + "0.8": 25, + "0.9": 24 + }, + "babel_mu1_abs": 0.3868374824523926 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 32, + "l0_raw": 31.8907474609375, + "recon_r2": 0.909052910263614, + "l1_per_token": 316.5885294143599, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 17, + "dense_rate": 0.0027669270833333335, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 681, + "0.6": 242, + "0.7": 73, + "0.8": 44, + "0.9": 36 + }, + "n_dup_clusters_signed": { + "0.5": 310, + "0.6": 99, + "0.7": 18, + "0.8": 4, + "0.9": 3 + }, + "largest_dup_signed": { + "0.5": 38, + "0.6": 38, + "0.7": 38, + "0.8": 38, + "0.9": 32 + }, + "babel_mu1_signed": 0.37396326661109924, + "n_dup_abs": { + "0.5": 799, + "0.6": 341, + "0.7": 152, + "0.8": 99, + "0.9": 74 + }, + "n_dup_clusters_abs": { + "0.5": 367, + "0.6": 148, + "0.7": 57, + "0.8": 31, + "0.9": 22 + }, + "largest_dup_abs": { + "0.5": 38, + "0.6": 38, + "0.7": 38, + "0.8": 38, + "0.9": 32 + }, + "babel_mu1_abs": 0.3880908787250519 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 32, + "l0_raw": 31.83146630859375, + "recon_r2": 0.9089463530749562, + "l1_per_token": 311.8740459469568, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 22, + "dense_rate": 0.0035807291666666665, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 659, + "0.6": 211, + "0.7": 85, + "0.8": 45, + "0.9": 35 + }, + "n_dup_clusters_signed": { + "0.5": 304, + "0.6": 87, + "0.7": 26, + "0.8": 7, + "0.9": 4 + }, + "largest_dup_signed": { + "0.5": 34, + "0.6": 34, + "0.7": 34, + "0.8": 33, + "0.9": 20 + }, + "babel_mu1_signed": 0.3730832636356354, + "n_dup_abs": { + "0.5": 757, + "0.6": 300, + "0.7": 151, + "0.8": 95, + "0.9": 72 + }, + "n_dup_clusters_abs": { + "0.5": 350, + "0.6": 129, + "0.7": 57, + "0.8": 31, + "0.9": 22 + }, + "largest_dup_abs": { + "0.5": 34, + "0.6": 34, + "0.7": 34, + "0.8": 33, + "0.9": 20 + }, + "babel_mu1_abs": 0.38568219542503357 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8203732967376709, + "dcs_abs": 0.8216442465782166, + "frac_sign_flipped": 0.015950520833333332, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8148746490478516, + "0.6": 0.8201885223388672, + "0.7": 0.8207994103431702, + "0.8": 0.8214713931083679, + "0.9": 0.821065366268158 + }, + "unique_frac_signed": { + "0.5": 0.9420572916666666, + "0.6": 0.979248046875, + "0.7": 0.9916178385416666, + "0.8": 0.99462890625, + "0.9": 0.9954427083333334 + }, + "dedup_dcs_signed": { + "0.5": 0.7676586049298445, + "0.6": 0.803168008569628, + "0.7": 0.8139193371607689, + "0.8": 0.8170591932430398, + "0.9": 0.8173235319166755 + }, + "matched_frac_signed": { + "0.5": 0.8181966145833334, + "0.6": 0.8206380208333334, + "0.7": 0.78466796875, + "0.8": 0.7127278645833334, + "0.9": 0.5623372395833334 + }, + "matched_cos_abs": { + "0.5": 0.8168478608131409, + "0.6": 0.822627067565918, + "0.7": 0.8229872584342957, + "0.8": 0.8228278160095215, + "0.9": 0.8224746584892273 + }, + "unique_frac_abs": { + "0.5": 0.9317220052083334, + "0.6": 0.9713541666666666, + "0.7": 0.9851888020833334, + "0.8": 0.9896647135416666, + "0.9": 0.9921061197916666 + }, + "dedup_dcs_abs": { + "0.5": 0.7610751268269572, + "0.6": 0.7990622296929359, + "0.7": 0.8107978312667304, + "0.8": 0.8143236548251782, + "0.9": 0.8159821420607234 + }, + "matched_frac_abs": { + "0.5": 0.8116861979166666, + "0.6": 0.81640625, + "0.7": 0.7815755208333334, + "0.8": 0.7097981770833334, + "0.9": 0.5615234375 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8162028789520264, + "dcs_abs": 0.817528247833252, + "frac_sign_flipped": 0.016927083333333332, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8100764155387878, + "0.6": 0.8151828646659851, + "0.7": 0.8170379996299744, + "0.8": 0.8172552585601807, + "0.9": 0.8168050050735474 + }, + "unique_frac_signed": { + "0.5": 0.943359375, + "0.6": 0.9807942708333334, + "0.7": 0.9912923177083334, + "0.8": 0.9947916666666666, + "0.9": 0.99560546875 + }, + "dedup_dcs_signed": { + "0.5": 0.7641931810649112, + "0.6": 0.7995266833459027, + "0.7": 0.8099234923089778, + "0.8": 0.8129987207551796, + "0.9": 0.8132155299535953 + }, + "matched_frac_signed": { + "0.5": 0.8181966145833334, + "0.6": 0.8185221354166666, + "0.7": 0.7784830729166666, + "0.8": 0.7000325520833334, + "0.9": 0.55419921875 + }, + "matched_cos_abs": { + "0.5": 0.8112047910690308, + "0.6": 0.8169951438903809, + "0.7": 0.8185408711433411, + "0.8": 0.8183879852294922, + "0.9": 0.8183701634407043 + }, + "unique_frac_abs": { + "0.5": 0.9337565104166666, + "0.6": 0.97314453125, + "0.7": 0.9852701822916666, + "0.8": 0.989990234375, + "0.9": 0.9922688802083334 + }, + "dedup_dcs_abs": { + "0.5": 0.7574677549418993, + "0.6": 0.795054356334731, + "0.7": 0.8064839133245792, + "0.8": 0.810196113307029, + "0.9": 0.8120432456732185 + }, + "matched_frac_abs": { + "0.5": 0.81201171875, + "0.6": 0.8146158854166666, + "0.7": 0.7762044270833334, + "0.8": 0.6982421875, + "0.9": 0.5533854166666666 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8149011135101318, + "dcs_abs": 0.8166163563728333, + "frac_sign_flipped": 0.017252604166666668, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.808275043964386, + "0.6": 0.8141347169876099, + "0.7": 0.8145653009414673, + "0.8": 0.8147961497306824, + "0.9": 0.8147222399711609 + }, + "unique_frac_signed": { + "0.5": 0.94091796875, + "0.6": 0.978271484375, + "0.7": 0.99072265625, + "0.8": 0.99365234375, + "0.9": 0.9947916666666666 + }, + "dedup_dcs_signed": { + "0.5": 0.760520512558287, + "0.6": 0.7964447780686896, + "0.7": 0.8070082986378111, + "0.8": 0.8096241038583685, + "0.9": 0.8104788949713111 + }, + "matched_frac_signed": { + "0.5": 0.8141276041666666, + "0.6": 0.8154296875, + "0.7": 0.7771809895833334, + "0.8": 0.6979166666666666, + "0.9": 0.5432942708333334 + }, + "matched_cos_abs": { + "0.5": 0.8111812472343445, + "0.6": 0.8169897794723511, + "0.7": 0.8167878985404968, + "0.8": 0.817298412322998, + "0.9": 0.8169680833816528 + }, + "unique_frac_abs": { + "0.5": 0.9317220052083334, + "0.6": 0.9703776041666666, + "0.7": 0.984619140625, + "0.8": 0.9892578125, + "0.9": 0.99169921875 + }, + "dedup_dcs_abs": { + "0.5": 0.7557954182605803, + "0.6": 0.7927885848330334, + "0.7": 0.8042249987338437, + "0.8": 0.8085188395343721, + "0.9": 0.81018661003327 + }, + "matched_frac_abs": { + "0.5": 0.8087565104166666, + "0.6": 0.8131510416666666, + "0.7": 0.77587890625, + "0.8": 0.6980794270833334, + "0.9": 0.5428059895833334 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 19, + "n_dense_std": 2.5166114784235836, + "l0_mean": 32, + "l0_std": 0.07258501414699992, + "recon_r2_mean": 0.9090530339073636, + "recon_r2_std": null, + "l1_per_token_mean": 314.91522348273855, + "l1_per_token_std": 2.6381609120196323, + "babel_mu1_signed_mean": 0.37308626373608905, + "babel_mu1_signed_std": 0.0008755066799605991, + "n_dup_signed_per_threshold": { + "0.5": 659.0, + "0.6": 215.66666666666666, + "0.7": 75.66666666666667, + "0.8": 39.333333333333336, + "0.9": 31.666666666666668 + }, + "n_dup_signed_band": 90.58333333333333, + "n_dup_signed_band_std": 10.308774579615822, + "n_dup_clusters_signed_per_threshold": { + "0.5": 303.3333333333333, + "0.6": 89.33333333333333, + "0.7": 21.666666666666668, + "0.8": 4.666666666666667, + "0.9": 2.6666666666666665 + }, + "n_dup_clusters_signed_band": 29.583333333333336, + "n_dup_clusters_signed_band_std": 2.4537386440559095, + "largest_dup_signed_per_threshold": { + "0.5": 32.333333333333336, + "0.6": 32.333333333333336, + "0.7": 32.333333333333336, + "0.8": 32.0, + "0.9": 25.333333333333332 + }, + "largest_dup_signed_band": 30.5, + "largest_dup_signed_band_std": 5.878988008152423, + "dcs_signed_mean": 0.817159096399943, + "dcs_signed_std": 0.002858664229096372, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7641240995176809, + "0.6": 0.7997131566614067, + "0.7": 0.810283709369186, + "0.8": 0.8132273392855293, + "0.9": 0.8136726522805273 + }, + "dedup_dcs_signed_band": 0.8092242143991624, + "dedup_dcs_signed_band_std": 0.003499437001262478, + "matched_cos_signed_per_threshold": { + "0.5": 0.8110753695170084, + "0.6": 0.816502034664154, + "0.7": 0.8174675703048706, + "0.8": 0.8178409337997437, + "0.9": 0.8175308704376221 + }, + "matched_cos_signed_band": 0.8173353523015976, + "matched_cos_signed_band_std": 0.0032319305279758207, + "unique_frac_signed_per_threshold": { + "0.5": 0.9421115451388888, + "0.6": 0.9794379340277778, + "0.7": 0.9912109375, + "0.8": 0.9943576388888888, + "0.9": 0.9952799479166666 + }, + "unique_frac_signed_band": 0.9900716145833333, + "unique_frac_signed_band_std": 0.0006462557037056265, + "matched_frac_signed_per_threshold": { + "0.5": 0.8168402777777778, + "0.6": 0.8181966145833334, + "0.7": 0.7801106770833334, + "0.8": 0.7035590277777778, + "0.9": 0.5532769097222223 + }, + "matched_frac_signed_band": 0.7137858072916667, + "matched_frac_signed_band_std": 0.005879825648291299, + "babel_mu1_abs_mean": 0.38687018553415936, + "babel_mu1_abs_std": 0.0012046746152445304, + "n_dup_abs_per_threshold": { + "0.5": 773.0, + "0.6": 308.3333333333333, + "0.7": 149.66666666666666, + "0.8": 96.0, + "0.9": 71.33333333333333 + }, + "n_dup_abs_band": 156.33333333333334, + "n_dup_abs_band_std": 9.385272150200725, + "n_dup_clusters_abs_per_threshold": { + "0.5": 357.6666666666667, + "0.6": 134.0, + "0.7": 57.666666666666664, + "0.8": 32.333333333333336, + "0.9": 22.333333333333332 + }, + "n_dup_clusters_abs_band": 61.583333333333336, + "n_dup_clusters_abs_band_std": 2.553592241007427, + "largest_dup_abs_per_threshold": { + "0.5": 32.333333333333336, + "0.6": 32.333333333333336, + "0.7": 32.333333333333336, + "0.8": 32.0, + "0.9": 25.333333333333332 + }, + "largest_dup_abs_band": 30.5, + "largest_dup_abs_band_std": 5.878988008152423, + "dcs_abs_mean": 0.8185962835947672, + "dcs_abs_std": 0.002678702161542788, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.758112766676479, + "0.6": 0.7956350569535667, + "0.7": 0.8071689144417178, + "0.8": 0.8110128692221931, + "0.9": 0.8127373325890707 + }, + "dedup_dcs_abs_band": 0.8066385433016371, + "dedup_dcs_abs_band_std": 0.0031144191815304155, + "matched_cos_abs_per_threshold": { + "0.5": 0.813077966372172, + "0.6": 0.8188706636428833, + "0.7": 0.8194386760393778, + "0.8": 0.8195047378540039, + "0.9": 0.8192709684371948 + }, + "matched_cos_abs_band": 0.8192712614933649, + "matched_cos_abs_band_std": 0.0030414190629281766, + "unique_frac_abs_per_threshold": { + "0.5": 0.9324001736111112, + "0.6": 0.9716254340277777, + "0.7": 0.9850260416666666, + "0.8": 0.9896375868055555, + "0.9": 0.9920247395833334 + }, + "unique_frac_abs_band": 0.9845784505208333, + "unique_frac_abs_band_std": 0.0005900065104166852, + "matched_frac_abs_per_threshold": { + "0.5": 0.810818142361111, + "0.6": 0.814724392361111, + "0.7": 0.7778862847222223, + "0.8": 0.7020399305555557, + "0.9": 0.5525716145833334 + }, + "matched_frac_abs_band": 0.7118055555555557, + "matched_frac_abs_band_std": 0.005030839526393286, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/config.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..805139f825bcdfb5940e33cea037c7c7a87efe38 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3ed1986b8f4d0b303d810a449097654061289966 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f653704b6233b079c695e35684cf2bfa40f8fcc57f8d4fa101ca61e4e1172cc +size 37776772 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..60bcc8b5bbb62ccd36e6327242bef5bb97c8d241 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19b7d42a92c5fba6091a4f3c864dc6e75636afc2eb8a29d2cfeeeaeedcaa090d +size 50496 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e5905406cffbbb6aafcfa462917ddfecf7c3b47 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfd53c0ec7308f13dd422b412eff32a6079cda5afc39818ccf7ce1ceeee0707b +size 25895 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..46ae2483e62d33916daf2a611778efb4ca1858f2 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.9758677734375, + "l0_per_token_0.001": 31.9758677734375, + "l0_per_token_0.01": 31.9758677734375, + "l0_per_token_0.05": 31.9758677734375, + "l0_per_token_0.1": 31.9758677734375, + "l0_per_token_0.5": 31.9758677734375, + "l0_per_token_1.0": 31.9758677734375, + "l1_per_token": 316.2830950868989, + "r2_score": 0.9091598383835207, + "l2_loss": 2.130422879907869 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..316c8e647c1bdd3bb5da046bd3fabc0d2b97f50f --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49c63200a858d371944e9a64f7c8cbe202bb644177ba82727175bbff974faf9c +size 25806 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..22402e6f65eaf21e65a4ea54ed99d1508b36ad3f --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d31839d87953e996e7e26b2260fef51387ac21cb85cf85a168c443cc2f93b58f +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1d0978f983b2569039f8e732ae9257799f0b213 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de00bb220b4aa41a2014673ba9113d77c587489f972da8c8ddc407fa6b55de12 +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/config.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..80f1abe0c12e221d402f0d112333352fb502d1d0 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0e2d7b7f6c96e0f3d4c115028043716687f58c20 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2a62537dee3edafa8744a919db6c0532dd1a7f0c62a3e2056336e0ba18b62a0 +size 37776772 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0d1bde21fdb781f2a9d722d30625df841673860 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2085b82c32949c97bc6aa1b60251b81638f211755a9697e50f6a07ae77e614f +size 50496 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..52d3a9423a740bd6b2a4a9481b471b02976b4145 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3368ba37f20e4154f4c71a87e5d75e272e308f84e357f9175fc15739aeca95ab +size 25895 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b934ffff59b19cb6c1989023b65d67dedf84b357 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.8907474609375, + "l0_per_token_0.001": 31.8907474609375, + "l0_per_token_0.01": 31.8907474609375, + "l0_per_token_0.05": 31.8907474609375, + "l0_per_token_0.1": 31.8907474609375, + "l0_per_token_0.5": 31.8907474609375, + "l0_per_token_1.0": 31.8907474609375, + "l1_per_token": 316.5885294143599, + "r2_score": 0.909052910263614, + "l2_loss": 2.1329306045651233 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8fc6674d5459d3aade4f54e6ccb68e6d8afc1c3 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b016f3d1449e540fdd67d09ca04941bc539410fc5a65f06fd1f84273e86238a0 +size 25806 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba000582532f81ba3c724718b75c4b92fb31c397 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb5910ef1efb1a47ecf09457b148978f94352154c489e43de854f490f6ff8954 +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..97e14d241f1533e0f0edf1434c8bed173738e6ce --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d1c67b845acc8d1c082e6411162d2b61d3cf2e67a0413e246ba91d7f9d3f584 +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/config.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ccd49714f71afa3b581cadb1c2e688b8da4a4291 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c6c9f5e0798da0a2efa00f63f8ef674f49c549a6 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe11a5e0b8fcea18fd4e0ae6b2b42dda46202137126f9a470968882a08fd48f4 +size 37776772 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b36477b46773197c101a9ed4537894a1c86e90bc --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5aebd80da5deece59bba5de6b5e7819d1eaaafc9c8d90fb3db5b226bfd520c8c +size 50496 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..580742abacaa80e8b54a49485db35c33edb7a58c --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3f3c988c2ce464f705619505d6a50c2a1c71e8c4d15d14cef8982460635d8c7 +size 25895 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..45ce9ec63e2cc3cac38be30c2c845700b4e1f6b0 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.83146630859375, + "l0_per_token_0.001": 31.83146630859375, + "l0_per_token_0.01": 31.83146630859375, + "l0_per_token_0.05": 31.83146630859375, + "l0_per_token_0.1": 31.83146630859375, + "l0_per_token_0.5": 31.83146630859375, + "l0_per_token_1.0": 31.83146630859375, + "l1_per_token": 311.8740459469568, + "r2_score": 0.9089463530749562, + "l2_loss": 2.1354296299818056 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5345964473b8bd8d8b77df2ca471096d9141304a --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbdecc1220a450e8ad26f9b768923b5dea7329eb8458654eaa9b8ae0d719c1bf +size 25806 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7f53cb9c5e4352dd3053cb4c2b0b749b6a49a66 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:666a28870d77b153ee36ea31e5bf58d9ea7e32b1fc229826ac2dc0b291984949 +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..26fa6a382d38fd13a152b66d65390c06b6a67798 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:424c7f865740bce0680d0f076dbe50a839e6fd2b0b62ebae0cf9a812cb466715 +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/code_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..cd41c59077d585b1aa3d1aac6e414bd47f7cc5a0 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 63.766488333333335, + "recon_r2": 0.9293372152654524, + "l1_per_token": 427.55473387960154 + }, + "seed1": { + "l0": 63.763515, + "recon_r2": 0.9293597846770042, + "l1_per_token": 429.14569312832555 + }, + "seed2": { + "l0": 63.503645, + "recon_r2": 0.9292783047294007, + "l1_per_token": 426.58760892299733 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7593626849291537, + "mcc_abs": 0.7593936528435118, + "act_iou": 0.43094143307549376, + "act_iou_n_pairs": 6144, + "input_r2": 0.969360272870769, + "matched_mcc_signed": { + "0.6": 0.8702479654543016, + "0.7": 0.891493226174305, + "0.8": 0.9144480636256229, + "0.9": 0.9408393701233216 + }, + "matched_mcc_abs": { + "0.6": 0.8672625058827911, + "0.7": 0.8897394826114445, + "0.8": 0.9136591344160849, + "0.9": 0.9407799274543017 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.760032368226172, + "mcc_abs": 0.7600646298948206, + "act_iou": 0.4319171413319862, + "act_iou_n_pairs": 6144, + "input_r2": 0.9691778356992782, + "matched_mcc_signed": { + "0.6": 0.8685185910565226, + "0.7": 0.8934003205148443, + "0.8": 0.9158927877177633, + "0.9": 0.9417286413380799 + }, + "matched_mcc_abs": { + "0.6": 0.8663904718787188, + "0.7": 0.8928536581806609, + "0.8": 0.915527224259374, + "0.9": 0.9415148479043554 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7568520772179358, + "mcc_abs": 0.7568794726125551, + "act_iou": 0.42794483385367416, + "act_iou_n_pairs": 6144, + "input_r2": 0.969288151641457, + "matched_mcc_signed": { + "0.6": 0.8684224531599573, + "0.7": 0.8904625298448002, + "0.8": 0.914269907199407, + "0.9": 0.9399357684345205 + }, + "matched_mcc_abs": { + "0.6": 0.8654203041917706, + "0.7": 0.8886017898031692, + "0.8": 0.9132613711143739, + "0.9": 0.9398458047931659 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 63.677882777777775, + "l0_std": 0.15090166527073742, + "recon_r2_mean": 0.9293251015572856, + "recon_r2_std": 4.2069013650543925e-05, + "l1_per_token_mean": 427.7626786436415, + "l1_per_token_std": 1.2916576439041954, + "input_r2_mean": 0.9692754200705013, + "input_r2_std": 9.188253402926586e-05, + "mcc_signed_mean": 0.7587490434577538, + "mcc_signed_std": 0.001676597630868367, + "mcc_abs_mean": 0.7587792517836291, + "mcc_abs_std": 0.001679113821329865, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8690630032235939, + "0.7": 0.8917853588446499, + "0.8": 0.9148702528475977, + "0.9": 0.9408345932986407 + }, + "matched_mcc_signed_band": 0.9041383020536206, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8663577606510935, + "0.7": 0.8903983101984249, + "0.8": 0.9141492432632776, + "0.9": 0.9407135267172744 + }, + "matched_mcc_abs_band": 0.9029047102075176, + "act_iou_mean": 0.43026780275371806, + "act_iou_std": 0.002070057978264191, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/dict_metrics_step_final.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..d17961933eb8b68a0edd77ceb45853f586aad949 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 63.82721625976563, + "recon_r2": 0.9292386438439206, + "l1_per_token": 427.56393678631576, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 57, + "dense_rate": 0.00927734375, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 270, + "0.6": 83, + "0.7": 17, + "0.8": 3, + "0.9": 3 + }, + "n_dup_clusters_signed": { + "0.5": 130, + "0.6": 40, + "0.7": 8, + "0.8": 1, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 4, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 3 + }, + "babel_mu1_signed": 0.3079947233200073, + "n_dup_abs": { + "0.5": 595, + "0.6": 373, + "0.7": 271, + "0.8": 213, + "0.9": 137 + }, + "n_dup_clusters_abs": { + "0.5": 290, + "0.6": 184, + "0.7": 135, + "0.8": 106, + "0.9": 68 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 3 + }, + "babel_mu1_abs": 0.3473738133907318 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 63.81022578125, + "recon_r2": 0.9292598842761588, + "l1_per_token": 429.0838694786593, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 64, + "dense_rate": 0.010416666666666666, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 253, + "0.6": 86, + "0.7": 20, + "0.8": 2, + "0.9": 2 + }, + "n_dup_clusters_signed": { + "0.5": 124, + "0.6": 43, + "0.7": 10, + "0.8": 1, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_signed": 0.3059081435203552, + "n_dup_abs": { + "0.5": 553, + "0.6": 347, + "0.7": 261, + "0.8": 198, + "0.9": 116 + }, + "n_dup_clusters_abs": { + "0.5": 272, + "0.6": 173, + "0.7": 130, + "0.8": 99, + "0.9": 58 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.34347012639045715 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 63.56048520507812, + "recon_r2": 0.929167587728551, + "l1_per_token": 426.5549158934567, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 62, + "dense_rate": 0.010091145833333334, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 238, + "0.6": 75, + "0.7": 20, + "0.8": 4, + "0.9": 2 + }, + "n_dup_clusters_signed": { + "0.5": 116, + "0.6": 37, + "0.7": 10, + "0.8": 2, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 3, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_signed": 0.304991215467453, + "n_dup_abs": { + "0.5": 569, + "0.6": 378, + "0.7": 277, + "0.8": 216, + "0.9": 120 + }, + "n_dup_clusters_abs": { + "0.5": 280, + "0.6": 187, + "0.7": 138, + "0.8": 108, + "0.9": 60 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 4, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.34560373425483704 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.7797131538391113, + "dcs_abs": 0.7816243171691895, + "frac_sign_flipped": 0.021158854166666668, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.7769481539726257, + "0.6": 0.7787163257598877, + "0.7": 0.7797335982322693, + "0.8": 0.7798026204109192, + "0.9": 0.7798026204109192 + }, + "unique_frac_signed": { + "0.5": 0.9781087239583334, + "0.6": 0.9930013020833334, + "0.7": 0.9984537760416666, + "0.8": 0.999755859375, + "0.9": 0.999755859375 + }, + "dedup_dcs_signed": { + "0.5": 0.7599397674639476, + "0.6": 0.7732663254331177, + "0.7": 0.7785279554615651, + "0.8": 0.7796122389117954, + "0.9": 0.7796122389117954 + }, + "matched_frac_signed": { + "0.5": 0.8224283854166666, + "0.6": 0.7936197916666666, + "0.7": 0.73876953125, + "0.8": 0.6422526041666666, + "0.9": 0.4567057291666667 + }, + "matched_cos_abs": { + "0.5": 0.7858443856239319, + "0.6": 0.7867981195449829, + "0.7": 0.7864798307418823, + "0.8": 0.785805344581604, + "0.9": 0.784477710723877 + }, + "unique_frac_abs": { + "0.5": 0.9523111979166666, + "0.6": 0.970458984375, + "0.7": 0.978271484375, + "0.8": 0.9832356770833334, + "0.9": 0.9896647135416666 + }, + "dedup_dcs_abs": { + "0.5": 0.7483684082496135, + "0.6": 0.763555304001784, + "0.7": 0.76939079145086, + "0.8": 0.7726318500353955, + "0.9": 0.7763699088633681 + }, + "matched_frac_abs": { + "0.5": 0.81298828125, + "0.6": 0.78759765625, + "0.7": 0.7342122395833334, + "0.8": 0.6375325520833334, + "0.9": 0.4537760416666667 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.7809880375862122, + "dcs_abs": 0.7829489707946777, + "frac_sign_flipped": 0.02294921875, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.7775358557701111, + "0.6": 0.7798774242401123, + "0.7": 0.780920147895813, + "0.8": 0.7810742855072021, + "0.9": 0.7810782790184021 + }, + "unique_frac_signed": { + "0.5": 0.9786783854166666, + "0.6": 0.993408203125, + "0.7": 0.9984537760416666, + "0.8": 0.9996744791666666, + "0.9": 0.999755859375 + }, + "dedup_dcs_signed": { + "0.5": 0.7609575359286584, + "0.6": 0.7747366306721233, + "0.7": 0.7797126704535913, + "0.8": 0.7808200295548886, + "0.9": 0.7808875860791886 + }, + "matched_frac_signed": { + "0.5": 0.826171875, + "0.6": 0.7972005208333334, + "0.7": 0.7332356770833334, + "0.8": 0.6435546875, + "0.9": 0.4562174479166667 + }, + "matched_cos_abs": { + "0.5": 0.7848207354545593, + "0.6": 0.786684513092041, + "0.7": 0.7874400019645691, + "0.8": 0.786662757396698, + "0.9": 0.7853732705116272 + }, + "unique_frac_abs": { + "0.5": 0.95166015625, + "0.6": 0.9690755208333334, + "0.7": 0.9776204427083334, + "0.8": 0.9825032552083334, + "0.9": 0.989501953125 + }, + "dedup_dcs_abs": { + "0.5": 0.7468826237309258, + "0.6": 0.762356704256187, + "0.7": 0.769817443326853, + "0.8": 0.7728987198934192, + "0.9": 0.7771283851034241 + }, + "matched_frac_abs": { + "0.5": 0.8133138020833334, + "0.6": 0.7892252604166666, + "0.7": 0.7262369791666666, + "0.8": 0.63916015625, + "0.9": 0.4534505208333333 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.7776620984077454, + "dcs_abs": 0.7799422144889832, + "frac_sign_flipped": 0.022623697916666668, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.7748982310295105, + "0.6": 0.7769970297813416, + "0.7": 0.7776418328285217, + "0.8": 0.7777538895606995, + "0.9": 0.7776321768760681 + }, + "unique_frac_signed": { + "0.5": 0.9795735677083334, + "0.6": 0.993408203125, + "0.7": 0.9983723958333334, + "0.8": 0.999755859375, + "0.9": 0.9998372395833334 + }, + "dedup_dcs_signed": { + "0.5": 0.7590698247804539, + "0.6": 0.7718752231885446, + "0.7": 0.7763761397412358, + "0.8": 0.7775640082400059, + "0.9": 0.7775056091389464 + }, + "matched_frac_signed": { + "0.5": 0.822265625, + "0.6": 0.7897135416666666, + "0.7": 0.7301432291666666, + "0.8": 0.6324869791666666, + "0.9": 0.45166015625 + }, + "matched_cos_abs": { + "0.5": 0.7820057272911072, + "0.6": 0.7843074202537537, + "0.7": 0.7842944860458374, + "0.8": 0.7832807302474976, + "0.9": 0.7815080881118774 + }, + "unique_frac_abs": { + "0.5": 0.95361328125, + "0.6": 0.9702962239583334, + "0.7": 0.97802734375, + "0.8": 0.983154296875, + "0.9": 0.9903971354166666 + }, + "dedup_dcs_abs": { + "0.5": 0.7457310475583654, + "0.6": 0.7610105282947188, + "0.7": 0.7670614529051818, + "0.8": 0.770085815602215, + "0.9": 0.7740033717709593 + }, + "matched_frac_abs": { + "0.5": 0.8123372395833334, + "0.6": 0.7819010416666666, + "0.7": 0.7252604166666666, + "0.8": 0.6271158854166666, + "0.9": 0.4485677083333333 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 61, + "n_dense_std": 3.605551275463989, + "l0_mean": 64, + "l0_std": 0.1493343492980087, + "recon_r2_mean": 0.9292220386162101, + "recon_r2_std": null, + "l1_per_token_mean": 427.7342407194772, + "l1_per_token_std": 1.2730491775840027, + "babel_mu1_signed_mean": 0.30629802743593854, + "babel_mu1_signed_std": 0.0015392439559120535, + "n_dup_signed_per_threshold": { + "0.5": 253.66666666666666, + "0.6": 81.33333333333333, + "0.7": 19.0, + "0.8": 3.0, + "0.9": 2.3333333333333335 + }, + "n_dup_signed_band": 26.416666666666664, + "n_dup_signed_band_std": 1.1273124382057236, + "n_dup_clusters_signed_per_threshold": { + "0.5": 123.33333333333333, + "0.6": 40.0, + "0.7": 9.333333333333334, + "0.8": 1.3333333333333333, + "0.9": 1.0 + }, + "n_dup_clusters_signed_band": 12.916666666666668, + "n_dup_clusters_signed_band_std": 0.7216878364870322, + "largest_dup_signed_per_threshold": { + "0.5": 3.3333333333333335, + "0.6": 2.6666666666666665, + "0.7": 2.3333333333333335, + "0.8": 2.3333333333333335, + "0.9": 2.3333333333333335 + }, + "largest_dup_signed_band": 2.416666666666667, + "largest_dup_signed_band_std": 0.5204164998665332, + "dcs_signed_mean": 0.7794544299443563, + "dcs_signed_std": 0.0016779962440372186, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7599890427243533, + "0.6": 0.7732927264312619, + "0.7": 0.7782055885521307, + "0.8": 0.7793320922355633, + "0.9": 0.7793351447099769 + }, + "dedup_dcs_signed_band": 0.7775413879822333, + "dedup_dcs_signed_band_std": 0.0016150907001028277, + "matched_cos_signed_per_threshold": { + "0.5": 0.7764607469240824, + "0.6": 0.7785302599271139, + "0.7": 0.7794318596522013, + "0.8": 0.7795435984929403, + "0.9": 0.7795043587684631 + }, + "matched_cos_signed_band": 0.7792525192101797, + "matched_cos_signed_band_std": 0.0016314181833985448, + "unique_frac_signed_per_threshold": { + "0.5": 0.9787868923611112, + "0.6": 0.9932725694444445, + "0.7": 0.9984266493055555, + "0.8": 0.9997287326388888, + "0.9": 0.9997829861111112 + }, + "unique_frac_signed_band": 0.997802734375, + "unique_frac_signed_band_std": 5.382794822317053e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.8236219618055555, + "0.6": 0.7935112847222222, + "0.7": 0.7340494791666666, + "0.8": 0.639431423611111, + "0.9": 0.45486111111111116 + }, + "matched_frac_signed_band": 0.6554633246527778, + "matched_frac_signed_band_std": 0.0038671300721818905, + "babel_mu1_abs_mean": 0.34548255801200867, + "babel_mu1_abs_std": 0.001954662582237108, + "n_dup_abs_per_threshold": { + "0.5": 572.3333333333334, + "0.6": 366.0, + "0.7": 269.6666666666667, + "0.8": 209.0, + "0.9": 124.33333333333333 + }, + "n_dup_abs_band": 242.25000000000003, + "n_dup_abs_band_std": 10.18270592720815, + "n_dup_clusters_abs_per_threshold": { + "0.5": 280.6666666666667, + "0.6": 181.33333333333334, + "0.7": 134.33333333333334, + "0.8": 104.33333333333333, + "0.9": 62.0 + }, + "n_dup_clusters_abs_band": 120.5, + "n_dup_clusters_abs_band_std": 4.763139720814412, + "largest_dup_abs_per_threshold": { + "0.5": 3.6666666666666665, + "0.6": 3.3333333333333335, + "0.7": 3.0, + "0.8": 2.3333333333333335, + "0.9": 2.3333333333333335 + }, + "largest_dup_abs_band": 2.7500000000000004, + "largest_dup_abs_band_std": 0.25, + "dcs_abs_mean": 0.7815051674842834, + "dcs_abs_std": 0.001506915178773653, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.7469940265129682, + "0.6": 0.76230751218423, + "0.7": 0.768756562560965, + "0.8": 0.7718721285103433, + "0.9": 0.7758338885792505 + }, + "dedup_dcs_abs_band": 0.7696925229586973, + "dedup_dcs_abs_band_std": 0.0014312244031545703, + "matched_cos_abs_per_threshold": { + "0.5": 0.7842236161231995, + "0.6": 0.7859300176302592, + "0.7": 0.7860714395840963, + "0.8": 0.7852496107419332, + "0.9": 0.7837863564491272 + }, + "matched_cos_abs_band": 0.785259356101354, + "matched_cos_abs_band_std": 0.0016871464703223443, + "unique_frac_abs_per_threshold": { + "0.5": 0.9525282118055555, + "0.6": 0.969943576388889, + "0.7": 0.9779730902777778, + "0.8": 0.9829644097222223, + "0.9": 0.9898546006944443 + }, + "unique_frac_abs_band": 0.9801839192708334, + "unique_frac_abs_band_std": 0.00044153919299745083, + "matched_frac_abs_per_threshold": { + "0.5": 0.8128797743055557, + "0.6": 0.7862413194444443, + "0.7": 0.7285698784722222, + "0.8": 0.6346028645833334, + "0.9": 0.4519314236111111 + }, + "matched_frac_abs_band": 0.6503363715277778, + "matched_frac_abs_band_std": 0.004054811924788146, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/config.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1f7e162da1f7234843065d629bdbb933d9c48c76 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d140f6d166bc5810f7eb99f9e5d537c5175311ea --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:627c5fdaa2f68328fb1cda3f86e92efe756a78c0a8ff376c07a9db531e0827b1 +size 37776772 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..68d2040ea959f7b11d0c4d96c3b4140a3739e973 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb5d8cb71f7b2de7c121ef867eb2b4af4d8aafa414daf4ba5dd3a2933938630c +size 50496 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..21cc3ee1d0ca029709fbb282c1e53934fb4d1e25 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06ba1e6e0a03f9cd3fcd4abbf2667fbdce846474deba7144abd4ff40c1976196 +size 25895 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..f8b7b36759e4ada1da2bd46a2fa03666502f15e9 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.82721625976563, + "l0_per_token_0.001": 63.82721625976563, + "l0_per_token_0.01": 63.82721625976563, + "l0_per_token_0.05": 63.82721625976563, + "l0_per_token_0.1": 63.82721625976563, + "l0_per_token_0.5": 63.82721625976563, + "l0_per_token_1.0": 63.82721625976563, + "l1_per_token": 427.56393678631576, + "r2_score": 0.9292386438439206, + "l2_loss": 1.6595260233538935 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..197f39c53eae8fb02bd0c30435242adbe242b798 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94ef5605f7138821d3c7f23601275fa6fc885309a4cca226c629a9c04c4ec266 +size 25806 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a711469d2555231ff9f194a9de5a3a49030acc67 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccffad4553c89abeeb613b52736bc6396a8c5c935ff8d454c9f4cc2303eab7d3 +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..503a5cba117cca811e54ad0b19052489981ab538 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63a62fa6e511ed3e77aaf38147ea7403526865eef1bd424dcd357de0e5809812 +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/config.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..28c0933da485f4d5bb12131f345860d2a2df616d --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1a6988821b073a69061d06a66ce7450f7a1572b0 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16eaf559159d0d5aa54d4186228f408f9aebf4e2f1934e13ffdc6bf929bbb0b4 +size 37776772 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf53542f4acb1e104cbd970a017b05dddcd2d34b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aa25f4e5f29bb3337d52c9f19e007e4a5d40dfc04bc9e0b6279ff9aeb3f1f71 +size 50496 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..cdc13643b72a467cb3f572021a4108974c4dd47e --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e38550275690195aed65f7d1ff221e299d34673baca3c4bf1cb288fa5152bd53 +size 25895 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..0112d09b3ff73624b00e15eff8a81052754d0863 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.81022578125, + "l0_per_token_0.001": 63.81022578125, + "l0_per_token_0.01": 63.81022578125, + "l0_per_token_0.05": 63.81022578125, + "l0_per_token_0.1": 63.81022578125, + "l0_per_token_0.5": 63.81022578125, + "l0_per_token_1.0": 63.81022578125, + "l1_per_token": 429.0838694786593, + "r2_score": 0.9292598842761588, + "l2_loss": 1.659027883522191 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a298feb138575d352ad5633ad46de3ebcd1aec8 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd64d2ff1fa48c3e9faa571220bb41f14b457bc9fce23d7d9ae2888bf9f9ebcc +size 25806 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1e9c7e10b0c82f55f5144a6f6d983bfcf4b3de3 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73d05eff59b0b7909affe287e74f1f3b607603ebf6826afa5bc6bd8cb420e767 +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7989230a26853cdcd0961c037cd79ad0b4c9fb15 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98235b318d09e0ace48b9cc3300309884eb7569d5e43e29336e9e14e01829483 +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/config.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4130da590b75251f4fc34dbd2b8480e46e402163 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/model.safetensors b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..aa7ef833f95f96de72b0f3d6286f40b00e4c4b54 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7d7a5af70105249022cfe9f9e11d88b6f2f6244e81a1122356656663732be82 +size 37776772 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9dc473e52d3ceb22663c6e67110cb06b258388e --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58066d4ea7ab89e7d283a38cf5d93b38f9b9abc6d9267754298502cd62f296bc +size 50496 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f33762d4632fab6c7987df20e4e488a26ccf43b --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e1b685deee1064a7f4664419275f54f2ec4367467fedaf3ed1c8342116c92a2 +size 25895 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..7b9e8a2768b7bfa79012d080b8dea20cdd624ba1 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.56048520507812, + "l0_per_token_0.001": 63.56048520507812, + "l0_per_token_0.01": 63.56048520507812, + "l0_per_token_0.05": 63.56048520507812, + "l0_per_token_0.1": 63.56048520507812, + "l0_per_token_0.5": 63.56048520507812, + "l0_per_token_1.0": 63.56048520507812, + "l1_per_token": 426.5549158934567, + "r2_score": 0.929167587728551, + "l2_loss": 1.6611924621982004 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6bc48c2261e9617b287d8678e94c280e10d1ec5e --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88d007575974daed5ca85e0d8b456d0dd4ae715a788d5c7ff2291d071e373876 +size 25806 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..4391ea6b424d85af030bc50ecb4569fb403c0522 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8628224398253f1a66d80f6f77e01213f39e4c04d1aa33e8e21373d214281e2f +size 25811 diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b981688737a74665283a5f25895c4b9d6d6de3ad --- /dev/null +++ b/lm/gpt2-small_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c4bfd95f7b4640f7c7bc88e57c7ef9337e4657394232c715f5240905c7146f3 +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..497642d0f17c164e23f4b810cbe36acc7af96f5f --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6abbca61151d92f6ac409e35475d2f6579b7b578e037b333108d32df9c288291 +size 75550008 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..15fc72308a47ed71de3917671ae6d86e0ecfa0d5 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1d0e143d9ffbaf873485e1df60ad7dbfd0d8c3ca3214bbd4fe8924032938705 +size 99648 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc87e4788e046ec607798710712dcc01aed9958d --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d75c1773cbb6326cc0c6045e3e8ac0a2cfc99e1d57fb2f132ab365eb3d58cb72 +size 50471 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d4c41f1448fc6800984f72a627c5aa333a99538 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0b222be58086eb929f7ba5f6fddc4fe0dd33534404c4900a3220ba5cbd43d22 +size 50382 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..73b7ddc60c904b3d4ff966512e99091511560983 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a134c85208a10e01b12767d6586459bce6bbd5ab908842b750575530cdf88b9 +size 50387 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..024dee6cc4f1377ebcefef75bd970242571c4f3a --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d705fc24eecbf34bfaa271eecd2039d608374a40b28e4af1a7941d0806a83145 +size 50387 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8c864e097d419d8ea2bb3e93914e273a2bf93cc4 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d326d79179f4c76b64c0ae88fea22967c36020d2f536fd3255a0b0afbff7cded +size 75550008 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea0857e3c6168d423dfa51b46b7c5034d961f75e --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd9583ac85652d7c5d4291fad82d7d1553843166c0119fbf9956f17f773bc386 +size 50471 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3846a0e2807e5d7c97a8cdf73eb98fe14f722e5 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:733f63ee80d3bbe3cc4c7def7fe2150a9f36b6b8830eede052a6987c3af2cf5c +size 50382 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6f960dd94409602031cc6acab6ec4e96e953370 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41fac98583f1991dea70e42242721cce2b35518d58cc7efb5e7ce0d7c988777c +size 50387 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b25585304ac242ad77692f5496bd996df1098e14 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0778f8f906fd8939d475533111f53f06c38f2c17593f1043fd6f413c181bb9a +size 50387 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/model.safetensors b/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..71e13ffc035ae035a6f00aa5dde3b5374f1aa40a --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e338935d5040e72e7cc2b39a49b95eef0d465c8fd3d430d03ffe3c0de400fcf +size 75550008 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..15fc72308a47ed71de3917671ae6d86e0ecfa0d5 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1d0e143d9ffbaf873485e1df60ad7dbfd0d8c3ca3214bbd4fe8924032938705 +size 99648 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..0dc817b8e7446b7f3c885e3959eda8df1fb108d2 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c72161d9892852d531bfb7cf3bcfb450ac2eb2fd4152c64edd31805bd33e2297 +size 50471 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..122f53aebabbdc55e11acb833459e3ce8cddb4fb --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa471c4a64feeecaf1103c13d61795e03cff46ade9b1989ba92a745946f41c57 +size 50382 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b65cb4bdf1155527fbd0c50365488a4b60580423 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6fefdef8bb128215172db540b15b2e1ce0fc4b1d9cbabf104dab9de12b54a9b +size 50387 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..92d661e5b50092cc04c7d9f9976abf955a6ea867 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d918858764bbebfb32cb06a7011552c44d6449ae099a49560068b5ba90c366f8 +size 50387 diff --git a/lm/gpt2-small_layer8/isae/x16/l0_32/seed0/model.safetensors b/lm/gpt2-small_layer8/isae/x16/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e4848524fe10953978b309f3586e0e748c6b8923 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x16/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15b435f566fe52954f604b78b1e8eb87838c1d724e0f4eb3152b355d4d72228b +size 75550008 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..df4feb0b328235274461675bbbc31151b6b375ba --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9506258c62a5f2cc3d220bfaf96384534a6b0dd07d29249350f99940dfcfc562 +size 18890032 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..25369b4d8167f4401220acd8088693c0808dd520 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f8664b39f84a7961b40b0e3ccd646d478422d3cc05283b1ec8476516297458 +size 25920 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..93722fd300ae6e281be91777928340e8ee2d5a81 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11b912df809163ed30cf78b767c14ac35715a83df899f75e9ffa230a075b4a32 +size 13607 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d445def56874c2203e9a8934a7ca743e8baaaea --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50d6eb46abd411f524993b6e14ff632dd214a76e2d19d980df06a6aa85844acb +size 13518 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3fbe1651fb20c72b1b93d0d31bde39d475b48fb4 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ba354ecf2fe6f7ebffa8226ef03d7a59e54fe79addcd2534a6c93a978829118 +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f714db51738588e163fd068de7d29f6d8e972e2a --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeea514adf05e3b72a98e321cea76bbf913f5092260394e4d2f437958329c56b +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..796b9161c9c6fb6fbb5677139693ccdedd5aff16 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e05a585e2750600036030d20eb1530556f2197ff030b55b016af4c72d788bc76 +size 18890032 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..25369b4d8167f4401220acd8088693c0808dd520 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f8664b39f84a7961b40b0e3ccd646d478422d3cc05283b1ec8476516297458 +size 25920 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc86f472ef969b3085434428ce9af11620be1e85 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:878e5478c870eed32e0df31ef75f7d7de73157e02a7fb1bfd7e18365b7aa6e9a +size 13607 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..396415e8e280027a7055b4ba09fea84fee213485 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0175ba353b2c72acde89f7de77ad92ed0161e8b4a48fb01dbd8b9d2f75986aee +size 13518 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..958d73c3c3545dab0e69cacfc41e444df3739a10 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffbd9bb41aea9d3f83459d6a30ac63d699716ba5a31e2c244e07c6b0aa341ccc +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ece60920221544ad75fd6e77c41f63a47f2b4f12 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03f24b33ad531845f7164a8d40977b53b901939f97260768543e450b3540b2cf +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/model.safetensors b/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..924c72737cdfd8a4eefe34758d47e7128518e786 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16b1b16aacbd3e6bac1f92dd2bcae683e5c043e2df838e3db86685d08f76ea2c +size 18890032 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..25369b4d8167f4401220acd8088693c0808dd520 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f8664b39f84a7961b40b0e3ccd646d478422d3cc05283b1ec8476516297458 +size 25920 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..23cd8fbee66804762e8f71414b40882a433b3828 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee38ffa887777c3a66c85691bc034eb0e8d7a744b82fe4be35d10087ef6c8ea8 +size 13607 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2f88d20bd846b042f27e20325089482c2b73479 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00c4a3055861dbdcc604f000bc0faa16a100a0f3063ee63b760deb1b25550daf +size 13518 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..55c0ae5f430bcc5e20e5ea0880a1dd9e8a2ad881 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3510a49c44266144da58a83a6f4d39cb755dcbbc1e0f3fe99e125ce1559716c6 +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3935f089c5b52a4cf4e7e4efa211e233aef65126 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a576a71dc297ba53ce2812abeeb26d1388c52c92ea868761b21355a0ef2dfec5 +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/model.safetensors b/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..706bceb0a24e06a025dbcb1a97f95e60544ccf3d --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38458cb5dc0adc48e8eea26a17962728164ae31361a2d97be2b3af220fe2c600 +size 18890032 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..22476d226c364cbdda1c2808c14b7ca570e296dc --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8cf66876f6b155c9ad52c0309e49a57494f55abcd7294eb01e0be14019e09c +size 25920 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..ace383b03778524b452447686220b5ea13054416 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baa1ff42e625a9d4aeaff86998188745b818a4ba4bfe8011958ec17c38362acc +size 13607 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..27edd4e6eb6a7c0b517e52265c80ebcb5bd9d48a --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:673f4cb55d4e315647700eb9c51af784186e6c80abeed73941351f6a1cdac378 +size 13518 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa9bc6e72d8f0df92f0be2ae13a645a49e37f478 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c5fd17db7b3007df7786a71aa85ca75d10d2eb73e49d2d8b59faaaed8c4e940 +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..41ee04067c0c332d2999d547eb29cdb359a241e0 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09c58f54cb16abad81b2763933aad13b69eec955c269d4e484a4325fc02ec397 +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/model.safetensors b/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ded95f3f25e268d5978e16e5560cc6402e762066 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9141f46348c83202e98b25c134861d6f3af43389dd5e38c719248901c97d06bf +size 18890032 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..22476d226c364cbdda1c2808c14b7ca570e296dc --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8cf66876f6b155c9ad52c0309e49a57494f55abcd7294eb01e0be14019e09c +size 25920 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee63a39b530735d3ead9cfe63a3a5ac93b6785b8 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b2f503ba7aa11a2cc00bab91099ea9e4d1e7163cec0194213e5df89be34feed +size 13607 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9762b81119472d3aad684652d3f1669e0c3688e9 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22579ce7dcf25cc8942e7a3b58ec9e3b253656051dec2ef32d83f7b29a8046b0 +size 13518 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e16f7a33cfc426eb33c94277d678e3e94fdf1d45 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d614c33d7c0773d53885fd387be8e24fe5118786535994f812727a45de2e5919 +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe1d64d8aecf948c2671e4a0aa763c6f3cc65f96 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3873a3ed9c157fa967f2e92dfd5f26022c45affb17cc3aece33b1e142778dbee +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/model.safetensors b/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1e4670a249685cbfe73149142eb684c35ecc98fb --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7769f0d3e27b8ee84a72e45c164347b1037fa7d6a99bf483db575d4f3e7bba4d +size 18890032 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..22476d226c364cbdda1c2808c14b7ca570e296dc --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8cf66876f6b155c9ad52c0309e49a57494f55abcd7294eb01e0be14019e09c +size 25920 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a52dc08cbd8ffae4a0765d1c793686c4c9d9938 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6629397182f922fead4cdeecab6d7d3f8a65f46910226cf0158fb4098addcfd8 +size 13607 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..76fa8daef13dc491ea7294aa10231052a93feae2 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8331dbbe0260efd818f08fda581ec6f068284c01196d1845f212b9df92fd0b2a +size 13518 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a539c8a3455fe225b21eff1ca4bce08bf896201e --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:034b1d87ae78b6715b531b8c6963b3308d8c483bf8f84b74af343f613b2991ff +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6194e2d4992ca1050795519ba403df04bd07e48 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83c70eb4aa5e1521b741dc6581ca5447ff8a6022a548b53a4376367346ae46a6 +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/model.safetensors b/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2422d019db1ad05021dacfd8c8abccb7d8dd7f62 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c16dc47cc9acfc04b064d399821ab7b8f0093773b9019eb08a87462c32f79a2 +size 18890032 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..05342aec6090b6546064ba3d08efbd5678ed2326 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d32f23fd81286996eebbb066afd1b434695b0b36f36a328d7242c2e286a4e12 +size 25920 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..ecec6892f97b51fb44baecf81597d7e8f9dbb0b2 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e54ed3bf9ef1bb10ff184e320142d2429251ce1ed161c0f394c4f3c2dc42b63 +size 13607 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d044113b9c22649f008b7ceeb79dc5ac9b24dec2 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:553166e1d1b52889d011d11909c357663d61392a3b99950f0e8a6405865cc140 +size 13518 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..326872cdff220ce49085b1c6b6de7259a4de71dc --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:776e3c6b1435c8ab02c7c4b6ee2c53cbb85e4aa7f2c6e5b237f22cf9f357b2d6 +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb6214f1673f22528e9263e058cbd9ef6514915c --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d82625119fefce8ac6f6131aca41b486b997388e45306a64b850c1f7f244dde +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/model.safetensors b/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3c42790a45a60bd54affd135242c11e81d822017 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5184b2d313fda0f2477e5464934cde22505d0421f584088dd57ad917316f4a12 +size 18890032 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..05342aec6090b6546064ba3d08efbd5678ed2326 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d32f23fd81286996eebbb066afd1b434695b0b36f36a328d7242c2e286a4e12 +size 25920 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca996a7ee2181619ad92099cd651b3abbc93c048 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31d345cd902d893896703b6ce405be54b710eb8dac00914424b6bb1ff898c789 +size 13607 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3dc26131b35f2cacc461462ddeea83d3497f511f --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a36854cd67dd29af156fa507dfc8154ef5cccc01d1574c754ba3177a085e0361 +size 13518 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..303edd6bc7839e3ad72b7e8963d168b0de4f5e8f --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c68c778ae328c5ff6e84ecc7fa14ef20e74a1e9884664cbd2e9669b70bb68db +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..327c3c7e62d9a54668d62fb5e5bfb314bd4c4123 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a98cfd7728c57972c620188508bc34aa93d0aa8ffa499a8076778177b5f2333 +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/model.safetensors b/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8f7a4851c6500a0e1c57f665aba392a83c538758 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6894e42a1e00f83441fbcb8eb29ab9b1e0913c52cc82d1525712accb03f00ff +size 18890032 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..05342aec6090b6546064ba3d08efbd5678ed2326 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d32f23fd81286996eebbb066afd1b434695b0b36f36a328d7242c2e286a4e12 +size 25920 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..4be61ab5250e1e52e5139b92dc918a09a62e65f5 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53828a3dab1d604034503369f0f9733880165bfce15551f17a08333ebb9ac68c +size 13607 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0cb27c2b77c3f0529bba7672ffec6b683c8fd06 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f42a402f868adf5c8c9ebb180d375ab2b5a5e9734aa8bf77965a4ac7d1cc4904 +size 13518 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f575f777bee2ddf682711ab5858b5b68a14ab53 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a6cc4ef5270cd5edebe4201929b02d0b96ccf44ced649159796159bbde21613 +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ceae55a405db68843b0232bed9e3d22e0d14ebb --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f86247ab04184f181771d0c344434dbdc5ae21f2856f2b95529b3ac7503d44b5 +size 13523 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..732f6a4946f9d0432ed9370bdde6c8babe9229a4 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3593e339d7efd67d7cb431d84d537475d7a2a0b90ba31bdbf81a0f47cc2f89fd +size 37776696 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..da8bb52901fc7e8720225d22c98d91c5121ddac6 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62cb2841c9a88a1c8f93b83e30c4831c7f45af5fff1fed059fc426b953645adf +size 50496 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..eef7fe7dd39c76ae6c0df1bdaffed1f71e8f6e03 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4f126a990d4d5642a8de8863eedab969e74a530ccac6df6b37fda0e52dc28fc +size 25895 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac57eeb3fd0796efc51d28c7ed55fdd5859160d7 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9870c000b73564aeaac8c69a757118ae31c1ba33e2fc0ad85c54c95b6cc01d5 +size 25806 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6299a92596f86327a15092d65667df5b18ca528d --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c7270062a01d6af2018b2377d07834c311f4852e13eca67591736bcdc33a8c1 +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f53deac39f11bffa6a8ae51f091c35a67d49616a --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b669473ca2a34b8de7da427b338e6511a0c360c6e1ecb09a6dd4d50e2cf004f5 +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ff4b00caaaf20b27e5d9ab047e1eb6ee709f6579 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c86386c148896525bf27fc5fcd3c001e0ab1441343c3c52b63dea592c695ed8 +size 37776696 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..da8bb52901fc7e8720225d22c98d91c5121ddac6 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62cb2841c9a88a1c8f93b83e30c4831c7f45af5fff1fed059fc426b953645adf +size 50496 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..a9f86af21a3f1dfce8449e715291f51e8feb3280 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d5cb57272adc0989499f26e1a9153f6abd522a1639140e1e443558b2e1afb35 +size 25895 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b931058e8a2f78819316fbfb77aba2814ef99c30 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:663593adcdb74091abed32663f045d78899c18abe7ba2647f482e99618a7b96d +size 25806 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..479488234a725ac3adad8a6668514cd2ecafe1b5 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f3f2831e0cd8abf3886527d876433bfd94d8d301ae48bd45c26fde7708fd91c +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..4374f1deb12e6cae53db9269f5d57b596cb34e57 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8ab2bc1302f7a2a95dc307671866aa9cf798fcd3bfa7a25939a74218c5ceb9a +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/model.safetensors b/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b37a722705336265b2bee0a51ea695da6a9df0e7 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c7e80b63355329c69c8ea163532b87a23a9bf405dc040aa89fbb7f191fe6c95 +size 37776696 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..da8bb52901fc7e8720225d22c98d91c5121ddac6 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62cb2841c9a88a1c8f93b83e30c4831c7f45af5fff1fed059fc426b953645adf +size 50496 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..693c2fde53742c4ef3fcd787bb89ded34ddb94a9 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:544de7491f34708537cdbd8d47c2b70b63b5ed3c93a91fadc0153647f73a2722 +size 25895 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8051abc889b0ccd1231ee2ac75cf29fa2363413 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc0e47e42c4c41318dda19c6e1553fc00a7878874e520158a263a151f7e7e0cd +size 25806 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1d26f4d6d30a882c6f22298fdb5940c75a2fc1b --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33b305022e50cd8ba08be73b405437f7f97273f60e47641e40176302019bd8c2 +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..fcbfd1e2d86556ba06bfa434c465a871c393d859 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:314b586f60edb98a532c20b63f1260612301c25cf0c7a3c117f9e69b54cf6e29 +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/model.safetensors b/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f593cb42f71f2427f8a09fd9293ea6fe7850803f --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ee7079a85849ea270a1642c6a4e3150164b99730e408b1fbd762af880df48c5 +size 37776696 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d846a9c18af168b9ab640277af5d8fd537dd811 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd44067a3b156c4c657c2ac2d0d08d2df65ac472c009e7ae3a381d80cb26c91 +size 50496 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f0cd30a2a2c90d710091caaa9cb6432b301954f --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fab817222949cfc208cc816351ebd6dd31e65b44ea95ab6bb94078b3fef8595 +size 25895 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..49b805bd99b5a7bd7ee14672b579b9d3b61f3e25 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea3efa6febc7ba7a4b0799a467a4ed67eb6ebfd710890cc234a518e4adf4ec5b +size 25806 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f453a755ed78cc3d196e0251b54a12de7332909b --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2200483ffd8f24b6005474e16d17ca8df5586adb7a2f7675dd727063fcbbd3d +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..402db11e9d67686c127e255afd1fc94576edd7e1 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f01c21ed75f7aeab51e8825afadf32fc40c0329f51603a041bddf0270f60575e +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/model.safetensors b/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..18049cbe73dcb15e938c58a88a0ec2bc5e73e3e8 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e551bdd7873d756f2b9fb4c3afc2f8369b14070eac3af1ee464b4212543fad41 +size 37776696 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d846a9c18af168b9ab640277af5d8fd537dd811 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd44067a3b156c4c657c2ac2d0d08d2df65ac472c009e7ae3a381d80cb26c91 +size 50496 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..2659c3e6a4e1d7bee1c2638d12e0d23779d25f9a --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d652fa213e7c7f0045e3dd7c4d0ca231955be866bd1b5403e7fecb75b2f91e4d +size 25895 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..157a93bb3cdbb9c04fe7f63b767e459eff5ef99d --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ee4c6da4618c98c1f1345ed36e3d3fd699e028fb8780b9fdbbff43d35c459f7 +size 25806 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8443ced6076f20ed5919327d78560ac64dc3b51 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c88530498e202d82758b7682473e1b27a972bb96fcbf94b7b0a2bb3814013280 +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..93aa298e031bf0969e5e11e26203ae2bf9d0a356 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31e1780fd0a87012d06ef7515840c8cdfff965dfb29e68446c0c65ee6b8a3a71 +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/model.safetensors b/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3077a149da06c63f4d33178d2dbf42587c440d97 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0621a963f7c55581da4e007c941aa35e47115a984e37476db72b438576e08967 +size 37776696 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d846a9c18af168b9ab640277af5d8fd537dd811 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd44067a3b156c4c657c2ac2d0d08d2df65ac472c009e7ae3a381d80cb26c91 +size 50496 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e5d999384953dbfed1d695efa23c8782e6e2b43 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dd597b620109b443fc2209545e231c98181ee2234dcbff3568e2f3a3461434c +size 25895 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d50a620945e2213c2f457721a47ece6ffd4cae8d --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7c6296b16b58d849699a45f8aa03670f21d4264e36723a02160531c66ba26b7 +size 25806 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b1dc794f937659926743ad0f18931b3728f5823d --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2da9d4800ca55788de7f1410be4cd99f79fe4518910c3232d5a5cc9e527184f4 +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4e1364ab788f6f2cfc993fe6efbcb383604fc27 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8494dbddfd10e0ac70ee0778d860db82cee2361c2795cfb8f64f9f27548093f +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/model.safetensors b/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8b2ee74a50d9b01537d1fe5bb39a80fea8029a83 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:165dd9420a6f019791bafaae8bfc18bd5b8a598f6bfcce2cab20a4ed24280899 +size 37776696 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..abcd360e514b4464f4c5e0612bc68c71a618868d --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f384933d03e973cacb959093fb9e9bb2cf7197ece15e4b731d8942192fa0463 +size 50496 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1547eeedf0576a57a2ef6674ed7410976dba484 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11c45d9411ae78efda1ae36fca773a363fb2f93ae0176a4d7dbab8b9b62f2773 +size 25895 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c912e5b3280b0b3cd30ef8d45f7cef6194517451 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2a9de1823f71f434288e7eb4db7bd0f275b9ab9da08c39c7264b2ece3fdf0b4 +size 25806 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5bbba6b4a4c2f3a16d32ea952a8a05d872360af1 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:542e9736b3a1628d2c1432781525b032c45b8c092bdd66291efa15a84b2a5974 +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..106a4238755e153c703322719b204888776cbec8 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46b289a6d0fff9b89560fb43dff909d501f43e2fed33cd1de5eee84172338125 +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/model.safetensors b/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2a9ba8159cfca777b50041af5dd7a7281267afc6 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86a9bc920454b11b047c9c842d92773769df63c7bbcef9fd2b2c3d711e83a524 +size 37776696 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..abcd360e514b4464f4c5e0612bc68c71a618868d --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f384933d03e973cacb959093fb9e9bb2cf7197ece15e4b731d8942192fa0463 +size 50496 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..118b3e60d887b4490c6c6fb1baf1d3bb59469d30 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ee62d252e7b703187da4a52cee7d2ec0aceee9c7c82eb2b8050c891b1224c54 +size 25895 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8abfff6aebbad8f039293c64c31ea38702dc86d --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a452c4f10828f0fe5b4e1ba04ccaefbde95b96f43b43e8af61d0eeaf3d064bdb +size 25806 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..84e33b6a09416be39859a5a2259ee5e09cc20240 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b079467733737b6c84394c33d07ce0e9c3764c8cb4177ac365e76d7e5007a6e +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1aef26dede3ec87d1b0cfc2837f4a51b92f5a0f1 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bdacac59c031f82cf177f1656e01eea7485c5ffa63b2d3c780bd4f3044ad01f +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/model.safetensors b/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..08ccc09315cd2ec3cc89eadb64f6f15ab668ff00 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f648cfb92e2e80bebe98f1803bc37937bc2de1ef1dc46942389ea809a774e157 +size 37776696 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..abcd360e514b4464f4c5e0612bc68c71a618868d --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f384933d03e973cacb959093fb9e9bb2cf7197ece15e4b731d8942192fa0463 +size 50496 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5871a2d4907cbef64587c78cf9c60a67c94be33 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b328c0d29b565ad5ce2481223b86af90ba7eda753402ec857e7101fe9f3b530 +size 25895 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..34a40b12865a1b60f0d83fee228907afcf6a0bf3 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3837fc944d5c10d95e0060294c028a3f825b229fc0458783af0715c13082d6ab +size 25806 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..667dc4763a72f0b95500fdd68e008e790ac60bcb --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:315801ef75dc4dd9235c0d8ec2f22b6097c0fd24c66730bf7ecc761232c49b14 +size 25811 diff --git a/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9eec7067982306032bf319d49135920629ecd4f9 --- /dev/null +++ b/lm/gpt2-small_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a99ec625dac8ab4ce94c493f24ac292b9a01d2344ebfa7029d323b3f7a9f18bd +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_128/code_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..3751b3f9e6b94b541cb4993d6a865f2256b4cddd --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 128.13204, + "recon_r2": 0.9478283255272012, + "l1_per_token": 733.4860440187557 + }, + "seed1": { + "l0": 127.64380166666666, + "recon_r2": 0.9478220670611325, + "l1_per_token": 734.5256281826365 + }, + "seed2": { + "l0": 127.79685, + "recon_r2": 0.9478615040848968, + "l1_per_token": 734.2379554029834 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.65314725429469, + "mcc_abs": 0.6531861394076898, + "act_iou": 0.30094024820133297, + "act_iou_n_pairs": 12288, + "input_r2": 0.956390125206105, + "matched_mcc_signed": { + "0.6": 0.8248875107717525, + "0.7": 0.854266742971009, + "0.8": 0.8818566814570058, + "0.9": 0.936416730715435 + }, + "matched_mcc_abs": { + "0.6": 0.8286534493040145, + "0.7": 0.8596157921732879, + "0.8": 0.8892228299352917, + "0.9": 0.9415142287027483 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6560865990266098, + "mcc_abs": 0.6561299091704575, + "act_iou": 0.3020061810566153, + "act_iou_n_pairs": 12288, + "input_r2": 0.9561674256839134, + "matched_mcc_signed": { + "0.6": 0.8254211860525704, + "0.7": 0.8550803276904939, + "0.8": 0.8817255612397605, + "0.9": 0.9381583358827075 + }, + "matched_mcc_abs": { + "0.6": 0.8287007852069107, + "0.7": 0.8603492174190702, + "0.8": 0.8892736417061655, + "0.9": 0.9432506171689933 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6538324586158372, + "mcc_abs": 0.6538709544598548, + "act_iou": 0.3018315553416676, + "act_iou_n_pairs": 12288, + "input_r2": 0.9565311623488751, + "matched_mcc_signed": { + "0.6": 0.8231668731191013, + "0.7": 0.8550536404878574, + "0.8": 0.8793974901665622, + "0.9": 0.9357552592440346 + }, + "matched_mcc_abs": { + "0.6": 0.8258780091691977, + "0.7": 0.8602453004385304, + "0.8": 0.8883278831670595, + "0.9": 0.9413805031466118 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 127.85756388888888, + "l0_std": 0.24971743984358574, + "recon_r2_mean": 0.9478372988910767, + "recon_r2_std": 2.1194590236182518e-05, + "l1_per_token_mean": 734.0832092014585, + "l1_per_token_std": 0.5367900880350939, + "input_r2_mean": 0.9563629044129646, + "input_r2_std": 0.000183389800817173, + "mcc_signed_mean": 0.654355437312379, + "mcc_signed_std": 0.0015378774011746302, + "mcc_abs_mean": 0.654395667679334, + "mcc_abs_std": 0.0015404343790206328, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8244918566478082, + "0.7": 0.8548002370497868, + "0.8": 0.8809932442877763, + "0.9": 0.9367767752807256 + }, + "matched_mcc_signed_band": 0.8742655283165242, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8277440812267076, + "0.7": 0.8600701033436295, + "0.8": 0.8889414516028388, + "0.9": 0.9420484496727844 + }, + "matched_mcc_abs_band": 0.8797010214614901, + "act_iou_mean": 0.3015926615332053, + "act_iou_std": 0.0005717131289237729, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_128/dict_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..93c31475c1b17b68bf2ee5533d3b29ffeb1227ae --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 128, + "l0_raw": 128.16427373046875, + "recon_r2": 0.9477530853883102, + "l1_per_token": 733.172180523432, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 451, + "dense_rate": 0.036702473958333336, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.17979370057582855, + "n_dup_abs": { + "0.5": 1006, + "0.6": 878, + "0.7": 830, + "0.8": 784, + "0.9": 696 + }, + "n_dup_clusters_abs": { + "0.5": 503, + "0.6": 439, + "0.7": 415, + "0.8": 392, + "0.9": 348 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2820708751678467 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 128, + "l0_raw": 127.66977475585938, + "recon_r2": 0.9477294159261017, + "l1_per_token": 734.230655681301, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 431, + "dense_rate": 0.035074869791666664, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.17976295948028564, + "n_dup_abs": { + "0.5": 1022, + "0.6": 908, + "0.7": 856, + "0.8": 814, + "0.9": 710 + }, + "n_dup_clusters_abs": { + "0.5": 511, + "0.6": 454, + "0.7": 428, + "0.8": 407, + "0.9": 355 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2833665907382965 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 128, + "l0_raw": 127.86142690429688, + "recon_r2": 0.9477797546293153, + "l1_per_token": 734.1147553738044, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 434, + "dense_rate": 0.035319010416666664, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.17975403368473053, + "n_dup_abs": { + "0.5": 1035, + "0.6": 914, + "0.7": 866, + "0.8": 822, + "0.9": 710 + }, + "n_dup_clusters_abs": { + "0.5": 517, + "0.6": 457, + "0.7": 433, + "0.8": 411, + "0.9": 355 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2825475335121155 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.5968829989433289, + "dcs_abs": 0.6005999445915222, + "frac_sign_flipped": 0.061197916666666664, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.5968311429023743, + "0.6": 0.5968829989433289, + "0.7": 0.5968829989433289, + "0.8": 0.5968829989433289, + "0.9": 0.5968829989433289 + }, + "unique_frac_signed": { + "0.5": 0.9998372395833334, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5967340024168759, + "0.6": 0.5968829989433289, + "0.7": 0.5968829989433289, + "0.8": 0.5968829989433289, + "0.9": 0.5968829989433289 + }, + "matched_frac_signed": { + "0.5": 0.726318359375, + "0.6": 0.5987955729166666, + "0.7": 0.3877766927083333, + "0.8": 0.146728515625, + "0.9": 0.033121744791666664 + }, + "matched_cos_abs": { + "0.5": 0.6029682755470276, + "0.6": 0.6030949950218201, + "0.7": 0.6029449105262756, + "0.8": 0.6030004620552063, + "0.9": 0.6022961735725403 + }, + "unique_frac_abs": { + "0.5": 0.958740234375, + "0.6": 0.9636637369791666, + "0.7": 0.9656982421875, + "0.8": 0.9674886067708334, + "0.9": 0.9713948567708334 + }, + "dedup_dcs_abs": { + "0.5": 0.5780899458186468, + "0.6": 0.581180776656159, + "0.7": 0.5822628402311238, + "0.8": 0.5833960769159603, + "0.9": 0.5850674052611188 + }, + "matched_frac_abs": { + "0.5": 0.709228515625, + "0.6": 0.583984375, + "0.7": 0.3767903645833333, + "0.8": 0.13997395833333334, + "0.9": 0.031005859375 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.5972462296485901, + "dcs_abs": 0.6009394526481628, + "frac_sign_flipped": 0.062174479166666664, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.5972049832344055, + "0.6": 0.5972462296485901, + "0.7": 0.5972462296485901, + "0.8": 0.5972462296485901, + "0.9": 0.5972462296485901 + }, + "unique_frac_signed": { + "0.5": 0.9998372395833334, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5971077819024989, + "0.6": 0.5972462296485901, + "0.7": 0.5972462296485901, + "0.8": 0.5972462296485901, + "0.9": 0.5972462296485901 + }, + "matched_frac_signed": { + "0.5": 0.7275390625, + "0.6": 0.5978190104166666, + "0.7": 0.384765625, + "0.8": 0.14469401041666666, + "0.9": 0.030354817708333332 + }, + "matched_cos_abs": { + "0.5": 0.6040030717849731, + "0.6": 0.6038243174552917, + "0.7": 0.6038090586662292, + "0.8": 0.6036956310272217, + "0.9": 0.6026715636253357 + }, + "unique_frac_abs": { + "0.5": 0.9584554036458334, + "0.6": 0.9635416666666666, + "0.7": 0.9654947916666666, + "0.8": 0.9673258463541666, + "0.9": 0.9713948567708334 + }, + "dedup_dcs_abs": { + "0.5": 0.5789100079709897, + "0.6": 0.5818098892147342, + "0.7": 0.5829745013033971, + "0.8": 0.5839703872237199, + "0.9": 0.5854320572276872 + }, + "matched_frac_abs": { + "0.5": 0.7111002604166666, + "0.6": 0.5840657552083334, + "0.7": 0.3740234375, + "0.8": 0.138427734375, + "0.9": 0.028727213541666668 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.5960850119590759, + "dcs_abs": 0.5998249650001526, + "frac_sign_flipped": 0.06315104166666667, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.5960570573806763, + "0.6": 0.5960850715637207, + "0.7": 0.5960850715637207, + "0.8": 0.5960850715637207, + "0.9": 0.5960850715637207 + }, + "unique_frac_signed": { + "0.5": 0.9998372395833334, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.59596004288566, + "0.6": 0.5960850715637207, + "0.7": 0.5960850715637207, + "0.8": 0.5960850715637207, + "0.9": 0.5960850715637207 + }, + "matched_frac_signed": { + "0.5": 0.7271321614583334, + "0.6": 0.5979817708333334, + "0.7": 0.3759765625, + "0.8": 0.14558919270833334, + "0.9": 0.031494140625 + }, + "matched_cos_abs": { + "0.5": 0.6021788120269775, + "0.6": 0.6019793152809143, + "0.7": 0.6019607782363892, + "0.8": 0.60200434923172, + "0.9": 0.601392388343811 + }, + "unique_frac_abs": { + "0.5": 0.9581298828125, + "0.6": 0.9629313151041666, + "0.7": 0.9649658203125, + "0.8": 0.9667154947916666, + "0.9": 0.9711100260416666 + }, + "dedup_dcs_abs": { + "0.5": 0.5769655145995785, + "0.6": 0.5796647337289565, + "0.7": 0.5808715761668282, + "0.8": 0.5819669323342774, + "0.9": 0.5840181779058184 + }, + "matched_frac_abs": { + "0.5": 0.708740234375, + "0.6": 0.5838216145833334, + "0.7": 0.3649088541666667, + "0.8": 0.13826497395833334, + "0.9": 0.029703776041666668 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 439, + "n_dense_std": 10.785793124908956, + "l0_mean": 128, + "l0_std": 0.24932441450608245, + "recon_r2_mean": 0.9477540853145757, + "recon_r2_std": null, + "l1_per_token_mean": 733.8391971928459, + "l1_per_token_std": 0.5805528815503181, + "babel_mu1_signed_mean": 0.17977023124694824, + "babel_mu1_signed_std": 2.0809240859340434e-05, + "n_dup_signed_per_threshold": { + "0.5": 4.0, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.0, + "n_dup_signed_band_std": 0.0, + "n_dup_clusters_signed_per_threshold": { + "0.5": 2.0, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.0, + "n_dup_clusters_signed_band_std": 0.0, + "largest_dup_signed_per_threshold": { + "0.5": 2.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.0, + "largest_dup_signed_band_std": 0.0, + "dcs_signed_mean": 0.5967380801836649, + "dcs_signed_std": 0.0005940182790007758, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5966006090683449, + "0.6": 0.5967381000518799, + "0.7": 0.5967381000518799, + "0.8": 0.5967381000518799, + "0.9": 0.5967381000518799 + }, + "dedup_dcs_signed_band": 0.5967381000518799, + "dedup_dcs_signed_band_std": 0.0005939855141935202, + "matched_cos_signed_per_threshold": { + "0.5": 0.596697727839152, + "0.6": 0.5967381000518799, + "0.7": 0.5967381000518799, + "0.8": 0.5967381000518799, + "0.9": 0.5967381000518799 + }, + "matched_cos_signed_band": 0.5967381000518799, + "matched_cos_signed_band_std": 0.0005939855141935202, + "unique_frac_signed_per_threshold": { + "0.5": 0.9998372395833334, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 1.0, + "unique_frac_signed_band_std": 0.0, + "matched_frac_signed_per_threshold": { + "0.5": 0.7269965277777778, + "0.6": 0.5981987847222222, + "0.7": 0.3828396267361111, + "0.8": 0.14567057291666666, + "0.9": 0.031656901041666664 + }, + "matched_frac_signed_band": 0.2895914713541667, + "matched_frac_signed_band_std": 0.0019291358190038001, + "babel_mu1_abs_mean": 0.28266166647275287, + "babel_mu1_abs_std": 0.0006553544532574925, + "n_dup_abs_per_threshold": { + "0.5": 1021.0, + "0.6": 900.0, + "0.7": 850.6666666666666, + "0.8": 806.6666666666666, + "0.9": 705.3333333333334 + }, + "n_dup_abs_band": 815.6666666666666, + "n_dup_abs_band_std": 16.441816606851365, + "n_dup_clusters_abs_per_threshold": { + "0.5": 510.3333333333333, + "0.6": 450.0, + "0.7": 425.3333333333333, + "0.8": 403.3333333333333, + "0.9": 352.6666666666667 + }, + "n_dup_clusters_abs_band": 407.8333333333333, + "n_dup_clusters_abs_band_std": 8.220908303425682, + "largest_dup_abs_per_threshold": { + "0.5": 2.3333333333333335, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.0, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.6004547874132792, + "dcs_abs_std": 0.0005712474369206918, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5779884894630717, + "0.6": 0.5808851331999499, + "0.7": 0.5820363059004497, + "0.8": 0.5831111321579859, + "0.9": 0.5848392134648748 + }, + "dedup_dcs_abs_band": 0.5827179461808151, + "dedup_dcs_abs_band_std": 0.000984046269925978, + "matched_cos_abs_per_threshold": { + "0.5": 0.6030500531196594, + "0.6": 0.6029662092526754, + "0.7": 0.6029049158096313, + "0.8": 0.6029001474380493, + "0.9": 0.602120041847229 + }, + "matched_cos_abs_band": 0.6027228285868963, + "matched_cos_abs_band_std": 0.0008385264915051927, + "unique_frac_abs_per_threshold": { + "0.5": 0.9584418402777778, + "0.6": 0.96337890625, + "0.7": 0.9653862847222222, + "0.8": 0.9671766493055555, + "0.9": 0.9712999131944445 + }, + "unique_frac_abs_band": 0.9668104383680556, + "unique_frac_abs_band_std": 0.00033450961521109, + "matched_frac_abs_per_threshold": { + "0.5": 0.7096896701388888, + "0.6": 0.583957248263889, + "0.7": 0.3719075520833333, + "0.8": 0.13888888888888892, + "0.9": 0.029812282986111115 + }, + "matched_frac_abs_band": 0.2811414930555556, + "matched_frac_abs_band_std": 0.0018876364013287582, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed0/config.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e3fd1b5739d61d18c1705071127e390e70d7b681 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..f0b0735eb70327b103d2b61be45a59f8b0914cbd --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.16427373046875, + "l0_per_token_0.001": 128.16427373046875, + "l0_per_token_0.01": 128.16427373046875, + "l0_per_token_0.05": 128.16427373046875, + "l0_per_token_0.1": 128.16427373046875, + "l0_per_token_0.5": 128.16427373046875, + "l0_per_token_1.0": 128.16427373046875, + "l1_per_token": 733.172180523432, + "r2_score": 0.9477530853883102, + "l2_loss": 1.2253173080346482 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed1/config.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..55945e3b61518cde4bc2bb08acdd35c4631f8042 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..117655cad14b333421fae8629856a06a00608118 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.66977475585938, + "l0_per_token_0.001": 127.66977475585938, + "l0_per_token_0.01": 127.66977475585938, + "l0_per_token_0.05": 127.66977475585938, + "l0_per_token_0.1": 127.66977475585938, + "l0_per_token_0.5": 127.66977475585938, + "l0_per_token_1.0": 127.66977475585938, + "l1_per_token": 734.230655681301, + "r2_score": 0.9477294159261017, + "l2_loss": 1.2258724145310098 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed2/config.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1689a729a9e579f9e7806492669b4a718217b682 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..05f9e75453f20a0554df70d49038575fc489fbb8 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.86142690429688, + "l0_per_token_0.001": 127.86142690429688, + "l0_per_token_0.01": 127.86142690429688, + "l0_per_token_0.05": 127.86142690429688, + "l0_per_token_0.1": 127.86142690429688, + "l0_per_token_0.5": 127.86142690429688, + "l0_per_token_1.0": 127.86142690429688, + "l1_per_token": 734.1147553738044, + "r2_score": 0.9477797546293153, + "l2_loss": 1.224691849424534 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_32/code_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..579624d045f0cc01ddc4b064771c6883c9bdd2a7 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 31.989943333333333, + "recon_r2": 0.9095528122985699, + "l1_per_token": 355.577257408727 + }, + "seed1": { + "l0": 31.898018333333333, + "recon_r2": 0.9094507319194027, + "l1_per_token": 356.86454958774567 + }, + "seed2": { + "l0": 31.937048333333333, + "recon_r2": 0.9094541021154816, + "l1_per_token": 350.60608747000697 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6674318647413108, + "mcc_abs": 0.6674533803577719, + "act_iou": 0.34766808405652566, + "act_iou_n_pairs": 12288, + "input_r2": 0.9564048357745951, + "matched_mcc_signed": { + "0.6": 0.8113820051055747, + "0.7": 0.840306808468379, + "0.8": 0.8730912287675545, + "0.9": 0.9091845777443003 + }, + "matched_mcc_abs": { + "0.6": 0.8089493206215156, + "0.7": 0.840139906431316, + "0.8": 0.8729800624091394, + "0.9": 0.9089776385651549 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6664355229665396, + "mcc_abs": 0.6664594700598248, + "act_iou": 0.34574831575174986, + "act_iou_n_pairs": 12288, + "input_r2": 0.9554641937674684, + "matched_mcc_signed": { + "0.6": 0.8124136456476807, + "0.7": 0.8413547182919586, + "0.8": 0.8716179542159648, + "0.9": 0.9088604758237153 + }, + "matched_mcc_abs": { + "0.6": 0.8105899276350855, + "0.7": 0.8407185738625496, + "0.8": 0.8713783775992078, + "0.9": 0.908195851880949 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6652971836550782, + "mcc_abs": 0.6653259868290431, + "act_iou": 0.3463698739191024, + "act_iou_n_pairs": 12288, + "input_r2": 0.955308213289265, + "matched_mcc_signed": { + "0.6": 0.811031753427582, + "0.7": 0.8412482808839701, + "0.8": 0.8706492650948328, + "0.9": 0.9057856050767922 + }, + "matched_mcc_abs": { + "0.6": 0.8081934529841931, + "0.7": 0.8404184573782296, + "0.8": 0.8709277761242873, + "0.9": 0.9054209893674612 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 31.941670000000002, + "l0_std": 0.04613644176064429, + "recon_r2_mean": 0.9094858821111513, + "recon_r2_std": 5.798773186033691e-05, + "l1_per_token_mean": 354.3492981554932, + "l1_per_token_std": 3.304996212629368, + "input_r2_mean": 0.9557257476104427, + "input_r2_std": 0.0005932562923509072, + "mcc_signed_mean": 0.6663881904543095, + "mcc_signed_std": 0.0010681273847324166, + "mcc_abs_mean": 0.6664129457488799, + "mcc_abs_std": 0.0010644595765697163, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8116091347269458, + "0.7": 0.8409699358814359, + "0.8": 0.8717861493594506, + "0.9": 0.9079435528816026 + }, + "matched_mcc_signed_band": 0.8580771932123588, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8092442337469313, + "0.7": 0.8404256458906983, + "0.8": 0.8717620720442115, + "0.9": 0.9075314932711883 + }, + "matched_mcc_abs_band": 0.8572408612382574, + "act_iou_mean": 0.3465954245757927, + "act_iou_std": 0.000979557252055994, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_32/dict_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..a376010e31ff59ee9c3630c69a8f2ba388beecc9 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 32.01708623046875, + "recon_r2": 0.9094417510755575, + "l1_per_token": 355.4332015009358, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 15, + "dense_rate": 0.001220703125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 55, + "0.6": 22, + "0.7": 18, + "0.8": 14, + "0.9": 12 + }, + "n_dup_clusters_signed": { + "0.5": 22, + "0.6": 6, + "0.7": 4, + "0.8": 2, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 12, + "0.6": 12, + "0.7": 12, + "0.8": 12, + "0.9": 12 + }, + "babel_mu1_signed": 0.22954244911670685, + "n_dup_abs": { + "0.5": 435, + "0.6": 212, + "0.7": 144, + "0.8": 98, + "0.9": 72 + }, + "n_dup_clusters_abs": { + "0.5": 211, + "0.6": 100, + "0.7": 66, + "0.8": 44, + "0.9": 31 + }, + "largest_dup_abs": { + "0.5": 12, + "0.6": 12, + "0.7": 12, + "0.8": 12, + "0.9": 12 + }, + "babel_mu1_abs": 0.2811225652694702 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 31.9189880859375, + "recon_r2": 0.9093025446652343, + "l1_per_token": 356.72511600793945, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 13, + "dense_rate": 0.0010579427083333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 47, + "0.6": 27, + "0.7": 14, + "0.8": 12, + "0.9": 12 + }, + "n_dup_clusters_signed": { + "0.5": 18, + "0.6": 8, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 13, + "0.6": 13, + "0.7": 12, + "0.8": 12, + "0.9": 12 + }, + "babel_mu1_signed": 0.22856467962265015, + "n_dup_abs": { + "0.5": 403, + "0.6": 237, + "0.7": 145, + "0.8": 108, + "0.9": 76 + }, + "n_dup_clusters_abs": { + "0.5": 195, + "0.6": 112, + "0.7": 67, + "0.8": 49, + "0.9": 33 + }, + "largest_dup_abs": { + "0.5": 14, + "0.6": 14, + "0.7": 12, + "0.8": 12, + "0.9": 12 + }, + "babel_mu1_abs": 0.27976182103157043 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 31.965524365234376, + "recon_r2": 0.9093196727242677, + "l1_per_token": 350.57077625032196, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 17, + "dense_rate": 0.0013834635416666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 52, + "0.6": 18, + "0.7": 12, + "0.8": 8, + "0.9": 8 + }, + "n_dup_clusters_signed": { + "0.5": 23, + "0.6": 6, + "0.7": 3, + "0.8": 1, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 8, + "0.6": 8, + "0.7": 8, + "0.8": 8, + "0.9": 8 + }, + "babel_mu1_signed": 0.22947090864181519, + "n_dup_abs": { + "0.5": 429, + "0.6": 226, + "0.7": 150, + "0.8": 94, + "0.9": 64 + }, + "n_dup_clusters_abs": { + "0.5": 210, + "0.6": 109, + "0.7": 71, + "0.8": 44, + "0.9": 29 + }, + "largest_dup_abs": { + "0.5": 8, + "0.6": 8, + "0.7": 8, + "0.8": 8, + "0.9": 8 + }, + "babel_mu1_abs": 0.2805631458759308 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.6803590655326843, + "dcs_abs": 0.68378084897995, + "frac_sign_flipped": 0.051595052083333336, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.6798688173294067, + "0.6": 0.6801609396934509, + "0.7": 0.6802343130111694, + "0.8": 0.6802085638046265, + "0.9": 0.6801695227622986 + }, + "unique_frac_signed": { + "0.5": 0.9974772135416666, + "0.6": 0.9985758463541666, + "0.7": 0.9989420572916666, + "0.8": 0.9990641276041666, + "0.9": 0.9991048177083334 + }, + "dedup_dcs_signed": { + "0.5": 0.678153653483605, + "0.6": 0.679192286011433, + "0.7": 0.6795146640797611, + "0.8": 0.6795719753863523, + "0.9": 0.6795606470501905 + }, + "matched_frac_signed": { + "0.5": 0.79736328125, + "0.6": 0.7176920572916666, + "0.7": 0.59033203125, + "0.8": 0.3866373697916667, + "0.9": 0.12906901041666666 + }, + "matched_cos_abs": { + "0.5": 0.6839330792427063, + "0.6": 0.6841638088226318, + "0.7": 0.6836435198783875, + "0.8": 0.683777928352356, + "0.9": 0.6837471723556519 + }, + "unique_frac_abs": { + "0.5": 0.982421875, + "0.6": 0.9903564453125, + "0.7": 0.99365234375, + "0.8": 0.9954020182291666, + "0.9": 0.99658203125 + }, + "dedup_dcs_abs": { + "0.5": 0.6719108180841431, + "0.6": 0.6775660377170425, + "0.7": 0.6793039858166594, + "0.8": 0.6806339299024936, + "0.9": 0.6814101458876394 + }, + "matched_frac_abs": { + "0.5": 0.7882486979166666, + "0.6": 0.7151692708333334, + "0.7": 0.5889485677083334, + "0.8": 0.3853352864583333, + "0.9": 0.12882486979166666 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.6786664128303528, + "dcs_abs": 0.6822295188903809, + "frac_sign_flipped": 0.052978515625, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.6783865094184875, + "0.6": 0.6786867380142212, + "0.7": 0.6787721514701843, + "0.8": 0.6787477135658264, + "0.9": 0.6786995530128479 + }, + "unique_frac_signed": { + "0.5": 0.9974772135416666, + "0.6": 0.9988606770833334, + "0.7": 0.9990641276041666, + "0.8": 0.9992268880208334, + "0.9": 0.999267578125 + }, + "dedup_dcs_signed": { + "0.5": 0.6766750851190105, + "0.6": 0.6779134946603639, + "0.7": 0.678136907350563, + "0.8": 0.6782229655776367, + "0.9": 0.6782024586136686 + }, + "matched_frac_signed": { + "0.5": 0.794677734375, + "0.6": 0.7151692708333334, + "0.7": 0.5865071614583334, + "0.8": 0.3880208333333333, + "0.9": 0.1240234375 + }, + "matched_cos_abs": { + "0.5": 0.6818870306015015, + "0.6": 0.6825457811355591, + "0.7": 0.6823472380638123, + "0.8": 0.6822386384010315, + "0.9": 0.6822412014007568 + }, + "unique_frac_abs": { + "0.5": 0.9819742838541666, + "0.6": 0.9906819661458334, + "0.7": 0.9936116536458334, + "0.8": 0.9957682291666666, + "0.9": 0.9969075520833334 + }, + "dedup_dcs_abs": { + "0.5": 0.6695955285443536, + "0.6": 0.6761857964399194, + "0.7": 0.6779881675732516, + "0.8": 0.679351560829673, + "0.9": 0.6801314060188209 + }, + "matched_frac_abs": { + "0.5": 0.78515625, + "0.6": 0.7120768229166666, + "0.7": 0.5855305989583334, + "0.8": 0.38720703125, + "0.9": 0.12369791666666667 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.678077757358551, + "dcs_abs": 0.6817903518676758, + "frac_sign_flipped": 0.052897135416666664, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.67779541015625, + "0.6": 0.6781810522079468, + "0.7": 0.6781591773033142, + "0.8": 0.6781555414199829, + "0.9": 0.6781555414199829 + }, + "unique_frac_signed": { + "0.5": 0.9976399739583334, + "0.6": 0.9987386067708334, + "0.7": 0.9991455078125, + "0.8": 0.999267578125, + "0.9": 0.999267578125 + }, + "dedup_dcs_signed": { + "0.5": 0.6761957953373592, + "0.6": 0.6773255992205426, + "0.7": 0.6775796955844271, + "0.8": 0.6776588454667944, + "0.9": 0.6776588454667944 + }, + "matched_frac_signed": { + "0.5": 0.791259765625, + "0.6": 0.7138671875, + "0.7": 0.5851236979166666, + "0.8": 0.3889973958333333, + "0.9": 0.130126953125 + }, + "matched_cos_abs": { + "0.5": 0.6817589402198792, + "0.6": 0.6819601058959961, + "0.7": 0.6818207502365112, + "0.8": 0.6818719506263733, + "0.9": 0.6818646192550659 + }, + "unique_frac_abs": { + "0.5": 0.9826253255208334, + "0.6": 0.9901529947916666, + "0.7": 0.9936116536458334, + "0.8": 0.9955647786458334, + "0.9": 0.996826171875 + }, + "dedup_dcs_abs": { + "0.5": 0.6699136005602971, + "0.6": 0.6752448411813626, + "0.7": 0.6774650431325426, + "0.8": 0.678847697590148, + "0.9": 0.6797004981490318 + }, + "matched_frac_abs": { + "0.5": 0.782958984375, + "0.6": 0.7113444010416666, + "0.7": 0.5840657552083334, + "0.8": 0.3876139322916667, + "0.9": 0.129638671875 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 15, + "n_dense_std": 2.0, + "l0_mean": 32, + "l0_std": 0.04907052271598234, + "recon_r2_mean": 0.9093546561550198, + "recon_r2_std": null, + "l1_per_token_mean": 354.24303125306574, + "l1_per_token_std": 3.245204674166818, + "babel_mu1_signed_mean": 0.2291926791270574, + "babel_mu1_signed_std": 0.0005450385702303479, + "n_dup_signed_per_threshold": { + "0.5": 51.333333333333336, + "0.6": 22.333333333333332, + "0.7": 14.666666666666666, + "0.8": 11.333333333333334, + "0.9": 10.666666666666666 + }, + "n_dup_signed_band": 14.75, + "n_dup_signed_band_std": 2.817356917396161, + "n_dup_clusters_signed_per_threshold": { + "0.5": 21.0, + "0.6": 6.666666666666667, + "0.7": 3.0, + "0.8": 1.3333333333333333, + "0.9": 1.0 + }, + "n_dup_clusters_signed_band": 3.0000000000000004, + "n_dup_clusters_signed_band_std": 0.25, + "largest_dup_signed_per_threshold": { + "0.5": 11.0, + "0.6": 11.0, + "0.7": 10.666666666666666, + "0.8": 10.666666666666666, + "0.9": 10.666666666666666 + }, + "largest_dup_signed_band": 10.749999999999998, + "largest_dup_signed_band_std": 2.384848003542364, + "dcs_signed_mean": 0.679034411907196, + "dcs_signed_std": 0.0011843391561372836, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.6770081779799915, + "0.6": 0.6781437932974465, + "0.7": 0.6784104223382504, + "0.8": 0.6784845954769279, + "0.9": 0.6784739837102179 + }, + "dedup_dcs_signed_band": 0.6783781987057107, + "dedup_dcs_signed_band_std": 0.0009781863485351593, + "matched_cos_signed_per_threshold": { + "0.5": 0.6786835789680481, + "0.6": 0.6790095766385397, + "0.7": 0.6790552139282227, + "0.8": 0.6790372729301453, + "0.9": 0.6790082057317098 + }, + "matched_cos_signed_band": 0.6790275673071544, + "matched_cos_signed_band_std": 0.0010481903068551234, + "unique_frac_signed_per_threshold": { + "0.5": 0.9975314670138888, + "0.6": 0.9987250434027778, + "0.7": 0.999050564236111, + "0.8": 0.9991861979166666, + "0.9": 0.9992133246527778 + }, + "unique_frac_signed_band": 0.9990437825520833, + "unique_frac_signed_band_std": 0.0001057159916729051, + "matched_frac_signed_per_threshold": { + "0.5": 0.79443359375, + "0.6": 0.715576171875, + "0.7": 0.5873209635416666, + "0.8": 0.38788519965277773, + "0.9": 0.1277398003472222 + }, + "matched_frac_signed_band": 0.45463053385416663, + "matched_frac_signed_band_std": 0.0012543182488442691, + "babel_mu1_abs_mean": 0.28048251072565716, + "babel_mu1_abs_std": 0.0006839464458852775, + "n_dup_abs_per_threshold": { + "0.5": 422.3333333333333, + "0.6": 225.0, + "0.7": 146.33333333333334, + "0.8": 100.0, + "0.9": 70.66666666666667 + }, + "n_dup_abs_band": 135.5, + "n_dup_abs_band_std": 5.291502622129181, + "n_dup_clusters_abs_per_threshold": { + "0.5": 205.33333333333334, + "0.6": 107.0, + "0.7": 68.0, + "0.8": 45.666666666666664, + "0.9": 31.0 + }, + "n_dup_clusters_abs_band": 62.916666666666664, + "n_dup_clusters_abs_band_std": 2.516611478423583, + "largest_dup_abs_per_threshold": { + "0.5": 11.333333333333334, + "0.6": 11.333333333333334, + "0.7": 10.666666666666666, + "0.8": 10.666666666666666, + "0.9": 10.666666666666666 + }, + "largest_dup_abs_band": 10.833333333333332, + "largest_dup_abs_band_std": 2.4664414311581235, + "dcs_abs_mean": 0.6826002399126688, + "dcs_abs_std": 0.001045751043875955, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6704733157295979, + "0.6": 0.6763322251127749, + "0.7": 0.6782523988408179, + "0.8": 0.6796110627741049, + "0.9": 0.6804140166851641 + }, + "dedup_dcs_abs_band": 0.6786524258532155, + "dedup_dcs_abs_band_std": 0.0009789819143904762, + "matched_cos_abs_per_threshold": { + "0.5": 0.6825263500213623, + "0.6": 0.6828898986180624, + "0.7": 0.6826038360595703, + "0.8": 0.6826295057932535, + "0.9": 0.6826176643371582 + }, + "matched_cos_abs_band": 0.6826852262020111, + "matched_cos_abs_band_std": 0.001020791099882263, + "unique_frac_abs_per_threshold": { + "0.5": 0.9823404947916666, + "0.6": 0.9903971354166666, + "0.7": 0.993625217013889, + "0.8": 0.9955783420138888, + "0.9": 0.9967719184027778 + }, + "unique_frac_abs_band": 0.9940931532118056, + "unique_frac_abs_band_std": 0.00013080038467063384, + "matched_frac_abs_per_threshold": { + "0.5": 0.7854546440972222, + "0.6": 0.7128634982638888, + "0.7": 0.586181640625, + "0.8": 0.38671875, + "0.9": 0.12738715277777776 + }, + "matched_frac_abs_band": 0.45328776041666663, + "matched_frac_abs_band_std": 0.001225272210686324, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed0/config.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..67cc265799484f523eb2550325684a58cd814b2e --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..191dc901840ce1a2c663bd692a840f993f7a47dd --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.01708623046875, + "l0_per_token_0.001": 32.01708623046875, + "l0_per_token_0.01": 32.01708623046875, + "l0_per_token_0.05": 32.01708623046875, + "l0_per_token_0.1": 32.01708623046875, + "l0_per_token_0.5": 32.01708623046875, + "l0_per_token_1.0": 32.01708623046875, + "l1_per_token": 355.4332015009358, + "r2_score": 0.9094417510755575, + "l2_loss": 2.1238113411505144 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed1/config.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec4bf09a23cb93d54f4886a0d5a1358212b2dd3b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..56a5b03a792333ca2593f07ee281df79f4ab43d4 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.9189880859375, + "l0_per_token_0.001": 31.9189880859375, + "l0_per_token_0.01": 31.9189880859375, + "l0_per_token_0.05": 31.9189880859375, + "l0_per_token_0.1": 31.9189880859375, + "l0_per_token_0.5": 31.9189880859375, + "l0_per_token_1.0": 31.9189880859375, + "l1_per_token": 356.72511600793945, + "r2_score": 0.9093025446652343, + "l2_loss": 2.1270760702780813 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed2/config.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c1789120be181f0a18e633a58926446a9bc1d5a1 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..4152bd8b09f27d4195d139b984a7ebe6f61f8301 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.965524365234376, + "l0_per_token_0.001": 31.965524365234376, + "l0_per_token_0.01": 31.965524365234376, + "l0_per_token_0.05": 31.965524365234376, + "l0_per_token_0.1": 31.965524365234376, + "l0_per_token_0.5": 31.965524365234376, + "l0_per_token_1.0": 31.965524365234376, + "l1_per_token": 350.57077625032196, + "r2_score": 0.9093196727242677, + "l2_loss": 2.126674375606872 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/code_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..9c23a70e87c5ac5725e4280234a4e694994a4d89 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 63.968228333333336, + "recon_r2": 0.9285323593396955, + "l1_per_token": 492.4976470414396 + }, + "seed1": { + "l0": 63.77768, + "recon_r2": 0.9285612204435997, + "l1_per_token": 490.89082453806003 + }, + "seed2": { + "l0": 63.86588666666667, + "recon_r2": 0.928604241835229, + "l1_per_token": 491.53842057018 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6650798216786925, + "mcc_abs": 0.665105701522875, + "act_iou": 0.3134689437819378, + "act_iou_n_pairs": 12288, + "input_r2": 0.9566825576588903, + "matched_mcc_signed": { + "0.6": 0.8196081202564781, + "0.7": 0.8498553704123705, + "0.8": 0.8802502245237315, + "0.9": 0.9242030675956524 + }, + "matched_mcc_abs": { + "0.6": 0.8189984973317185, + "0.7": 0.8505426476608675, + "0.8": 0.8814718030176154, + "0.9": 0.9249855415516083 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6643505730967764, + "mcc_abs": 0.6643805855666031, + "act_iou": 0.3139867623240838, + "act_iou_n_pairs": 12288, + "input_r2": 0.9565522336368684, + "matched_mcc_signed": { + "0.6": 0.8186722806565708, + "0.7": 0.8504022095786342, + "0.8": 0.8803622860682278, + "0.9": 0.928189455793815 + }, + "matched_mcc_abs": { + "0.6": 0.8165520346105186, + "0.7": 0.8500552577424855, + "0.8": 0.880778175073151, + "0.9": 0.9278849364346704 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6663342806423704, + "mcc_abs": 0.6663634924284868, + "act_iou": 0.3147188055715632, + "act_iou_n_pairs": 12288, + "input_r2": 0.9573956718459448, + "matched_mcc_signed": { + "0.6": 0.8215397055305748, + "0.7": 0.8516244089610974, + "0.8": 0.8842122134450728, + "0.9": 0.9262398103584935 + }, + "matched_mcc_abs": { + "0.6": 0.8201773149106198, + "0.7": 0.8517379667911668, + "0.8": 0.8843966551745671, + "0.9": 0.9262133385496039 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 63.870598333333334, + "l0_std": 0.09536150526345351, + "recon_r2_mean": 0.928565940539508, + "recon_r2_std": 3.617295633580605e-05, + "l1_per_token_mean": 491.64229738322655, + "l1_per_token_std": 0.8084320834543212, + "input_r2_mean": 0.9568768210472345, + "input_r2_std": 0.00045403821553996547, + "mcc_signed_mean": 0.6652548918059464, + "mcc_signed_std": 0.001003374839581023, + "mcc_abs_mean": 0.6652832598393217, + "mcc_abs_std": 0.0010033070928339586, + "matched_mcc_signed_per_threshold": { + "0.6": 0.819940035481208, + "0.7": 0.8506273296507008, + "0.8": 0.8816082413456773, + "0.9": 0.9262107779159869 + }, + "matched_mcc_signed_band": 0.8695965960983932, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8185759489509524, + "0.7": 0.8507786240648398, + "0.8": 0.8822155444217779, + "0.9": 0.9263612721786275 + }, + "matched_mcc_abs_band": 0.8694828474040495, + "act_iou_mean": 0.3140581705591949, + "act_iou_std": 0.0006279832603841097, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/dict_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..071b0640b71957d2ed774572c09be232f37bc568 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 63.99105180664063, + "recon_r2": 0.9284304754619364, + "l1_per_token": 492.3230827168743, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 54, + "dense_rate": 0.00439453125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 10, + "0.6": 2, + "0.7": 2, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 5, + "0.6": 1, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.19376635551452637, + "n_dup_abs": { + "0.5": 535, + "0.6": 373, + "0.7": 305, + "0.8": 228, + "0.9": 118 + }, + "n_dup_clusters_abs": { + "0.5": 267, + "0.6": 186, + "0.7": 152, + "0.8": 114, + "0.9": 59 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2618151903152466 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 63.78823364257813, + "recon_r2": 0.9284605175927734, + "l1_per_token": 490.6179402339079, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 56, + "dense_rate": 0.004557291666666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 4, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.19291190803050995, + "n_dup_abs": { + "0.5": 540, + "0.6": 388, + "0.7": 306, + "0.8": 246, + "0.9": 122 + }, + "n_dup_clusters_abs": { + "0.5": 270, + "0.6": 194, + "0.7": 153, + "0.8": 123, + "0.9": 61 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.26136311888694763 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 63.90943334960937, + "recon_r2": 0.9284991637617972, + "l1_per_token": 491.4820630568018, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 56, + "dense_rate": 0.004557291666666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 10, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 5, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.19304244220256805, + "n_dup_abs": { + "0.5": 524, + "0.6": 380, + "0.7": 294, + "0.8": 232, + "0.9": 108 + }, + "n_dup_clusters_abs": { + "0.5": 262, + "0.6": 190, + "0.7": 147, + "0.8": 116, + "0.9": 54 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.261501282453537 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.6413446068763733, + "dcs_abs": 0.6445313096046448, + "frac_sign_flipped": 0.054524739583333336, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.6412579417228699, + "0.6": 0.6413670778274536, + "0.7": 0.6413701772689819, + "0.8": 0.6413446068763733, + "0.9": 0.6413446068763733 + }, + "unique_frac_signed": { + "0.5": 0.9996337890625, + "0.6": 0.9999186197916666, + "0.7": 0.9999593098958334, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6410231060508522, + "0.6": 0.6413148832410418, + "0.7": 0.6413440798496595, + "0.8": 0.6413446068763733, + "0.9": 0.6413446068763733 + }, + "matched_frac_signed": { + "0.5": 0.7714029947916666, + "0.6": 0.6682942708333334, + "0.7": 0.5040690104166666, + "0.8": 0.264892578125, + "0.9": 0.06689453125 + }, + "matched_cos_abs": { + "0.5": 0.6455280184745789, + "0.6": 0.6456282734870911, + "0.7": 0.6453304290771484, + "0.8": 0.6452657580375671, + "0.9": 0.6446249485015869 + }, + "unique_frac_abs": { + "0.5": 0.9781087239583334, + "0.6": 0.9844970703125, + "0.7": 0.987548828125, + "0.8": 0.9903564453125, + "0.9": 0.9951171875 + }, + "dedup_dcs_abs": { + "0.5": 0.6313965864295218, + "0.6": 0.6356191437589587, + "0.7": 0.6372953089885414, + "0.8": 0.6390431024119607, + "0.9": 0.6414773657452315 + }, + "matched_frac_abs": { + "0.5": 0.7596842447916666, + "0.6": 0.6626790364583334, + "0.7": 0.5, + "0.8": 0.2621256510416667, + "0.9": 0.06624348958333333 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.6412110328674316, + "dcs_abs": 0.6446365714073181, + "frac_sign_flipped": 0.055989583333333336, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.6411324739456177, + "0.6": 0.6412514448165894, + "0.7": 0.6412559747695923, + "0.8": 0.6412110924720764, + "0.9": 0.6412110924720764 + }, + "unique_frac_signed": { + "0.5": 0.9995930989583334, + "0.6": 0.9999186197916666, + "0.7": 0.9999593098958334, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.640871596474123, + "0.6": 0.6411992596404161, + "0.7": 0.6412298819971815, + "0.8": 0.6412110924720764, + "0.9": 0.6412110924720764 + }, + "matched_frac_signed": { + "0.5": 0.7672526041666666, + "0.6": 0.6700846354166666, + "0.7": 0.5048014322916666, + "0.8": 0.266357421875, + "0.9": 0.06551106770833333 + }, + "matched_cos_abs": { + "0.5": 0.6458900570869446, + "0.6": 0.6459271311759949, + "0.7": 0.6458859443664551, + "0.8": 0.6453914642333984, + "0.9": 0.6449611783027649 + }, + "unique_frac_abs": { + "0.5": 0.9784342447916666, + "0.6": 0.9846598307291666, + "0.7": 0.98779296875, + "0.8": 0.9906412760416666, + "0.9": 0.9954020182291666 + }, + "dedup_dcs_abs": { + "0.5": 0.6319609502243111, + "0.6": 0.6360184996471313, + "0.7": 0.638001594459638, + "0.8": 0.6393514236745734, + "0.9": 0.6419956585620336 + }, + "matched_frac_abs": { + "0.5": 0.758056640625, + "0.6": 0.6659342447916666, + "0.7": 0.5025227864583334, + "0.8": 0.264404296875, + "0.9": 0.06510416666666667 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.6424317955970764, + "dcs_abs": 0.6457427740097046, + "frac_sign_flipped": 0.053955078125, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.6423835158348083, + "0.6": 0.6424057483673096, + "0.7": 0.6424317955970764, + "0.8": 0.6424317955970764, + "0.9": 0.6424317955970764 + }, + "unique_frac_signed": { + "0.5": 0.9996337890625, + "0.6": 0.9999186197916666, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6421482679652399, + "0.6": 0.6423534692536729, + "0.7": 0.6424317955970764, + "0.8": 0.6424317955970764, + "0.9": 0.6424317955970764 + }, + "matched_frac_signed": { + "0.5": 0.7705891927083334, + "0.6": 0.668212890625, + "0.7": 0.5052083333333334, + "0.8": 0.2677408854166667, + "0.9": 0.06892903645833333 + }, + "matched_cos_abs": { + "0.5": 0.6473837494850159, + "0.6": 0.6472585797309875, + "0.7": 0.6467761993408203, + "0.8": 0.6465396881103516, + "0.9": 0.6461108326911926 + }, + "unique_frac_abs": { + "0.5": 0.9783528645833334, + "0.6": 0.984375, + "0.7": 0.98779296875, + "0.8": 0.9902750651041666, + "0.9": 0.9953206380208334 + }, + "dedup_dcs_abs": { + "0.5": 0.6333697457933644, + "0.6": 0.6371451644226909, + "0.7": 0.6388809820637107, + "0.8": 0.640252131735906, + "0.9": 0.6430874462263698 + }, + "matched_frac_abs": { + "0.5": 0.76171875, + "0.6": 0.6648763020833334, + "0.7": 0.5018717447916666, + "0.8": 0.2659505208333333, + "0.9": 0.06827799479166667 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 55, + "n_dense_std": 1.1547005383792517, + "l0_mean": 64, + "l0_std": 0.10205076284730817, + "recon_r2_mean": 0.9284633856055023, + "recon_r2_std": null, + "l1_per_token_mean": 491.47436200252804, + "l1_per_token_std": 0.8525973266917296, + "babel_mu1_signed_mean": 0.19324023524920145, + "babel_mu1_signed_std": 0.0004602843607031304, + "n_dup_signed_per_threshold": { + "0.5": 9.333333333333334, + "0.6": 2.0, + "0.7": 0.6666666666666666, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.6666666666666666, + "n_dup_signed_band_std": 0.28867513459481287, + "n_dup_clusters_signed_per_threshold": { + "0.5": 4.666666666666667, + "0.6": 1.0, + "0.7": 0.3333333333333333, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.3333333333333333, + "n_dup_clusters_signed_band_std": 0.14433756729740643, + "largest_dup_signed_per_threshold": { + "0.5": 2.0, + "0.6": 2.0, + "0.7": 1.3333333333333333, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.3333333333333333, + "largest_dup_signed_band_std": 0.14433756729740646, + "dcs_signed_mean": 0.6416624784469604, + "dcs_signed_std": 0.0006695873072744889, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.6413476568300717, + "0.6": 0.6416225373783769, + "0.7": 0.6416685858146391, + "0.8": 0.6416624983151754, + "0.9": 0.6416624983151754 + }, + "dedup_dcs_signed_band": 0.6416540299558418, + "dedup_dcs_signed_band_std": 0.0006595373277768394, + "matched_cos_signed_per_threshold": { + "0.5": 0.6415913105010986, + "0.6": 0.6416747570037842, + "0.7": 0.6416859825452169, + "0.8": 0.6416624983151754, + "0.9": 0.6416624983151754 + }, + "matched_cos_signed_band": 0.6416714340448381, + "matched_cos_signed_band_std": 0.0006558006457090037, + "unique_frac_signed_per_threshold": { + "0.5": 0.9996202256944445, + "0.6": 0.9999186197916666, + "0.7": 0.999972873263889, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9999728732638888, + "unique_frac_signed_band_std": 5.873110648473362e-06, + "matched_frac_signed_per_threshold": { + "0.5": 0.7697482638888888, + "0.6": 0.6688639322916666, + "0.7": 0.5046929253472222, + "0.8": 0.2663302951388889, + "0.9": 0.06711154513888888 + }, + "matched_frac_signed_band": 0.37674967447916663, + "matched_frac_signed_band_std": 0.0007444732447726439, + "babel_mu1_abs_mean": 0.2615598638852437, + "babel_mu1_abs_std": 0.0002316591940257765, + "n_dup_abs_per_threshold": { + "0.5": 533.0, + "0.6": 380.3333333333333, + "0.7": 301.6666666666667, + "0.8": 235.33333333333334, + "0.9": 116.0 + }, + "n_dup_abs_band": 258.33333333333337, + "n_dup_abs_band_std": 6.331139971074193, + "n_dup_clusters_abs_per_threshold": { + "0.5": 266.3333333333333, + "0.6": 190.0, + "0.7": 150.66666666666666, + "0.8": 117.66666666666667, + "0.9": 58.0 + }, + "n_dup_clusters_abs_band": 129.08333333333331, + "n_dup_clusters_abs_band_std": 3.2145502536643185, + "largest_dup_abs_per_threshold": { + "0.5": 2.3333333333333335, + "0.6": 2.3333333333333335, + "0.7": 2.3333333333333335, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.166666666666667, + "largest_dup_abs_band_std": 0.2886751345948129, + "dcs_abs_mean": 0.6449702183405558, + "dcs_abs_std": 0.0006711197420922216, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6322424274823991, + "0.6": 0.6362609359429269, + "0.7": 0.63805929517063, + "0.8": 0.6395488859408134, + "0.9": 0.6421868235112116 + }, + "dedup_dcs_abs_band": 0.6390139851413954, + "dedup_dcs_abs_band_std": 0.0007561995761983201, + "matched_cos_abs_per_threshold": { + "0.5": 0.6462672750155131, + "0.6": 0.6462713281313578, + "0.7": 0.6459975242614746, + "0.8": 0.6457323034604391, + "0.9": 0.6452323198318481 + }, + "matched_cos_abs_band": 0.6458083689212799, + "matched_cos_abs_band_std": 0.0007652403625418461, + "unique_frac_abs_per_threshold": { + "0.5": 0.9782986111111112, + "0.6": 0.9845106336805555, + "0.7": 0.9877115885416666, + "0.8": 0.9904242621527777, + "0.9": 0.9952799479166666 + }, + "unique_frac_abs_band": 0.9894816080729166, + "unique_frac_abs_band_std": 0.00012705480953766562, + "matched_frac_abs_per_threshold": { + "0.5": 0.7598198784722222, + "0.6": 0.6644965277777778, + "0.7": 0.50146484375, + "0.8": 0.26416015625, + "0.9": 0.06654188368055557 + }, + "matched_frac_abs_band": 0.3741658528645833, + "matched_frac_abs_band_std": 0.0012726639138210605, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed0/config.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3e187a07c65dcae7ddde84a817fa7d9110867df5 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..64ef38d4c066ca0caa09f8f749e4fd154e07f54a --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.99105180664063, + "l0_per_token_0.001": 63.99105180664063, + "l0_per_token_0.01": 63.99105180664063, + "l0_per_token_0.05": 63.99105180664063, + "l0_per_token_0.1": 63.99105180664063, + "l0_per_token_0.5": 63.99105180664063, + "l0_per_token_1.0": 63.99105180664063, + "l1_per_token": 492.3230827168743, + "r2_score": 0.9284304754619364, + "l2_loss": 1.6784795388602454 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/config.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..abd948c4d6731bf89a82b1bc8e5174ba3cf4f99a --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d3ed44625c1d1dc7413fd79736289082c822d00 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f60152e6451b8ed79d1232b5cffe94dc98b49b46926c04847d76cb79333432a +size 99648 diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..100c1d95dda704798858de8cb8aace2265170d88 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42a897679b8c16a988d1ac6e3f72c92d61694793f5e5bc6d31b786aaaad75ce2 +size 50471 diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..984bd53ce6ac8fa8627a89e092dc8cc498d23cce --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.78823364257813, + "l0_per_token_0.001": 63.78823364257813, + "l0_per_token_0.01": 63.78823364257813, + "l0_per_token_0.05": 63.78823364257813, + "l0_per_token_0.1": 63.78823364257813, + "l0_per_token_0.5": 63.78823364257813, + "l0_per_token_1.0": 63.78823364257813, + "l1_per_token": 490.6179402339079, + "r2_score": 0.9284605175927734, + "l2_loss": 1.6777749777745141 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d53a88ecf20c695ba3ccaab437ea1ade4975578a --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1aa413b38722dacff6d3bf43538a78b292f01f3f4c90fa8cd34319c3fd1cd864 +size 50382 diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..73605ef83ba52144ed4ce6a70ec8a07fa1c7b34a --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bff8aedb4e2982544185d71230031fc1049aa036d6a672892ceefaaa135e40e1 +size 50387 diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..60bca9640a44d1b2f4c6176f5f798190a809488c --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc02fb09d48207b083374ea2b2dc2625ce04eb2f8dcaea882765057d18c8b04f +size 50387 diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/config.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c7856662ca76e8aa6b7e6e38f8b93b4b57f646de --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..faef392dc9fa069c765c544fb639ab7cc99e6eb0 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:118b14992fad4c20dba95570ad1578b0a4b2521c8b6d3534785494d2d3813403 +size 99648 diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..37fa1dbd576a9096b9ccf22bb8db1288f1108ebc --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd1aa5c05193f0a1770cc53fee34ad592b15e21138017408757bc0744b073f9e +size 50471 diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..a8dfa72689c5faef1fa0d4dc05eb9874709547f7 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.90943334960937, + "l0_per_token_0.001": 63.90943334960937, + "l0_per_token_0.01": 63.90943334960937, + "l0_per_token_0.05": 63.90943334960937, + "l0_per_token_0.1": 63.90943334960937, + "l0_per_token_0.5": 63.90943334960937, + "l0_per_token_1.0": 63.90943334960937, + "l1_per_token": 491.4820630568018, + "r2_score": 0.9284991637617972, + "l2_loss": 1.6768686310524932 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/code_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..6066852806208d9035aca0fb9b60e015c7b4ff12 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 128.30714, + "recon_r2": 0.930996280896217, + "l1_per_token": 810.7370429576766 + }, + "seed1": { + "l0": 127.67615333333333, + "recon_r2": 0.9306337657467254, + "l1_per_token": 810.4553098815283 + }, + "seed2": { + "l0": 127.86922333333334, + "recon_r2": 0.93083916044944, + "l1_per_token": 819.2539337084365 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6218091909535326, + "mcc_abs": 0.6218670890936963, + "act_iou": 0.2991519962963693, + "act_iou_n_pairs": 3072, + "input_r2": 0.9371104700051518, + "matched_mcc_signed": { + "0.6": 0.8143932751870363, + "0.7": 0.845191610325364, + "0.8": 0.8805430051831584, + "0.9": 0.9382361865043214 + }, + "matched_mcc_abs": { + "0.6": 0.8162457853144373, + "0.7": 0.8472261361310406, + "0.8": 0.885769833293564, + "0.9": 0.9446536824034039 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6156924655576536, + "mcc_abs": 0.615766928646657, + "act_iou": 0.2979994730870681, + "act_iou_n_pairs": 3072, + "input_r2": 0.9369912673274516, + "matched_mcc_signed": { + "0.6": 0.814282130845937, + "0.7": 0.8461654871363825, + "0.8": 0.8858391875357049, + "0.9": 0.9445479109907232 + }, + "matched_mcc_abs": { + "0.6": 0.8174084108000581, + "0.7": 0.8494760663195858, + "0.8": 0.8891163780039879, + "0.9": 0.9462834828789414 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6249104203353363, + "mcc_abs": 0.624975604870654, + "act_iou": 0.30132630627222984, + "act_iou_n_pairs": 3072, + "input_r2": 0.9373687511326038, + "matched_mcc_signed": { + "0.6": 0.818310034353365, + "0.7": 0.8452626055926019, + "0.8": 0.884746790090792, + "0.9": 0.9368064110349805 + }, + "matched_mcc_abs": { + "0.6": 0.8213421719432078, + "0.7": 0.847063170949395, + "0.8": 0.8900202053603924, + "0.9": 0.9429270878670611 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 127.9508388888889, + "l0_std": 0.32331388395684574, + "recon_r2_mean": 0.9308230690307941, + "recon_r2_std": 0.0001817924880688311, + "l1_per_token_mean": 813.4820955158806, + "l1_per_token_std": 5.000543017746734, + "input_r2_mean": 0.9371568294884024, + "input_r2_std": 0.00019296478189938885, + "mcc_signed_mean": 0.6208040256155075, + "mcc_signed_std": 0.0046904627264723366, + "mcc_abs_mean": 0.620869874203669, + "mcc_abs_std": 0.004684629932264702, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8156618134621128, + "0.7": 0.8455399010181162, + "0.8": 0.8837096609365518, + "0.9": 0.9398635028433416 + }, + "matched_mcc_signed_band": 0.8711937195650306, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8183321226859009, + "0.7": 0.8479217911333405, + "0.8": 0.8883021388859814, + "0.9": 0.9446214177164688 + }, + "matched_mcc_abs_band": 0.8747943676054228, + "act_iou_mean": 0.2994925918852224, + "act_iou_std": 0.0016893663832434382, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/dict_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..a1f0ea9caf7d6ec0b0fd75718c7aa3bcb250e7f4 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 128, + "l0_raw": 128.30693959960936, + "recon_r2": 0.9309044060817548, + "l1_per_token": 810.0086997059205, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 444, + "dense_rate": 0.14453125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.14485643804073334, + "n_dup_abs": { + "0.5": 584, + "0.6": 556, + "0.7": 534, + "0.8": 520, + "0.9": 476 + }, + "n_dup_clusters_abs": { + "0.5": 292, + "0.6": 278, + "0.7": 267, + "0.8": 260, + "0.9": 238 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.34475746750831604 + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 128, + "l0_raw": 127.6742666015625, + "recon_r2": 0.9305413569658916, + "l1_per_token": 809.9053929874115, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 442, + "dense_rate": 0.14388020833333334, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.14513321220874786, + "n_dup_abs": { + "0.5": 508, + "0.6": 480, + "0.7": 462, + "0.8": 450, + "0.9": 392 + }, + "n_dup_clusters_abs": { + "0.5": 254, + "0.6": 240, + "0.7": 231, + "0.8": 225, + "0.9": 196 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3306008279323578 + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 128, + "l0_raw": 127.89192016601562, + "recon_r2": 0.9307402745408487, + "l1_per_token": 818.6819381570849, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 420, + "dense_rate": 0.13671875, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.14562459290027618, + "n_dup_abs": { + "0.5": 586, + "0.6": 550, + "0.7": 534, + "0.8": 520, + "0.9": 460 + }, + "n_dup_clusters_abs": { + "0.5": 293, + "0.6": 275, + "0.7": 267, + "0.8": 260, + "0.9": 230 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.34606707096099854 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.5913465619087219, + "dcs_abs": 0.5939745306968689, + "frac_sign_flipped": 0.06803385416666667, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.5913301110267639, + "0.6": 0.5913465023040771, + "0.7": 0.5913465023040771, + "0.8": 0.5913465023040771, + "0.9": 0.5913465023040771 + }, + "unique_frac_signed": { + "0.5": 0.9998372395833334, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5912338658915057, + "0.6": 0.5913465023040771, + "0.7": 0.5913465023040771, + "0.8": 0.5913465023040771, + "0.9": 0.5913465023040771 + }, + "matched_frac_signed": { + "0.5": 0.6868489583333334, + "0.6": 0.59375, + "0.7": 0.4036458333333333, + "0.8": 0.20475260416666666, + "0.9": 0.056640625 + }, + "matched_cos_abs": { + "0.5": 0.6154040694236755, + "0.6": 0.6155833005905151, + "0.7": 0.6155253648757935, + "0.8": 0.6153669357299805, + "0.9": 0.6148199439048767 + }, + "unique_frac_abs": { + "0.5": 0.9111328125, + "0.6": 0.9156901041666666, + "0.7": 0.9189453125, + "0.8": 0.9210611979166666, + "0.9": 0.9293619791666666 + }, + "dedup_dcs_abs": { + "0.5": 0.5607148405979387, + "0.6": 0.5636835366409892, + "0.7": 0.5656341487774625, + "0.8": 0.5667906069817642, + "0.9": 0.5713902798985752 + }, + "matched_frac_abs": { + "0.5": 0.6614583333333334, + "0.6": 0.5751953125, + "0.7": 0.3912760416666667, + "0.8": 0.1953125, + "0.9": 0.054036458333333336 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.5892764329910278, + "dcs_abs": 0.5929126739501953, + "frac_sign_flipped": 0.080078125, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.5892298221588135, + "0.6": 0.5892763733863831, + "0.7": 0.5892763733863831, + "0.8": 0.5892763733863831, + "0.9": 0.5892763733863831 + }, + "unique_frac_signed": { + "0.5": 0.9998372395833334, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5891339188674465, + "0.6": 0.5892763733863831, + "0.7": 0.5892763733863831, + "0.8": 0.5892763733863831, + "0.9": 0.5892763733863831 + }, + "matched_frac_signed": { + "0.5": 0.6861979166666666, + "0.6": 0.58984375, + "0.7": 0.4153645833333333, + "0.8": 0.19694010416666666, + "0.9": 0.054036458333333336 + }, + "matched_cos_abs": { + "0.5": 0.6129291653633118, + "0.6": 0.6131632924079895, + "0.7": 0.6130968332290649, + "0.8": 0.6127349734306335, + "0.9": 0.6118685007095337 + }, + "unique_frac_abs": { + "0.5": 0.90478515625, + "0.6": 0.9099934895833334, + "0.7": 0.9130859375, + "0.8": 0.9153645833333334, + "0.9": 0.923828125 + }, + "dedup_dcs_abs": { + "0.5": 0.5545692106534261, + "0.6": 0.5579746041427522, + "0.7": 0.5598100967472419, + "0.8": 0.560875893648093, + "0.9": 0.5652613297570497 + }, + "matched_frac_abs": { + "0.5": 0.66015625, + "0.6": 0.572265625, + "0.7": 0.4039713541666667, + "0.8": 0.18977864583333334, + "0.9": 0.052408854166666664 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.5954223871231079, + "dcs_abs": 0.5985018014907837, + "frac_sign_flipped": 0.08138020833333333, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.5954224467277527, + "0.6": 0.5954224467277527, + "0.7": 0.5954224467277527, + "0.8": 0.5954224467277527, + "0.9": 0.5954224467277527 + }, + "unique_frac_signed": { + "0.5": 1.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5954224467277527, + "0.6": 0.5954224467277527, + "0.7": 0.5954224467277527, + "0.8": 0.5954224467277527, + "0.9": 0.5954224467277527 + }, + "matched_frac_signed": { + "0.5": 0.697265625, + "0.6": 0.5944010416666666, + "0.7": 0.4212239583333333, + "0.8": 0.20735677083333334, + "0.9": 0.055338541666666664 + }, + "matched_cos_abs": { + "0.5": 0.6207550168037415, + "0.6": 0.6205894947052002, + "0.7": 0.6207465529441833, + "0.8": 0.6206558346748352, + "0.9": 0.6189068555831909 + }, + "unique_frac_abs": { + "0.5": 0.9109700520833334, + "0.6": 0.9161783854166666, + "0.7": 0.9189453125, + "0.8": 0.9210611979166666, + "0.9": 0.9306640625 + }, + "dedup_dcs_abs": { + "0.5": 0.5654892299886949, + "0.6": 0.5685706812655553, + "0.7": 0.5704321350785904, + "0.8": 0.5716620065795723, + "0.9": 0.5759943685261533 + }, + "matched_frac_abs": { + "0.5": 0.6712239583333334, + "0.6": 0.5768229166666666, + "0.7": 0.4091796875, + "0.8": 0.19986979166666666, + "0.9": 0.0537109375 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 435, + "n_dense_std": 13.316656236958785, + "l0_mean": 128, + "l0_std": 0.3214263334050378, + "recon_r2_mean": 0.930728679196165, + "recon_r2_std": null, + "l1_per_token_mean": 812.8653436168056, + "l1_per_token_std": 5.037583479979067, + "babel_mu1_signed_mean": 0.14520474771658579, + "babel_mu1_signed_std": 0.0003890417313361713, + "n_dup_signed_per_threshold": { + "0.5": 0.6666666666666666, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.0, + "n_dup_signed_band_std": 0.0, + "n_dup_clusters_signed_per_threshold": { + "0.5": 0.3333333333333333, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.0, + "n_dup_clusters_signed_band_std": 0.0, + "largest_dup_signed_per_threshold": { + "0.5": 1.3333333333333333, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.0, + "largest_dup_signed_band_std": 0.0, + "dcs_signed_mean": 0.5920151273409525, + "dcs_signed_std": 0.003127046985782008, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.591930077162235, + "0.6": 0.5920151074727377, + "0.7": 0.5920151074727377, + "0.8": 0.5920151074727377, + "0.9": 0.5920151074727377 + }, + "dedup_dcs_signed_band": 0.5920151074727377, + "dedup_dcs_signed_band_std": 0.003127111931646074, + "matched_cos_signed_per_threshold": { + "0.5": 0.5919941266377767, + "0.6": 0.5920151074727377, + "0.7": 0.5920151074727377, + "0.8": 0.5920151074727377, + "0.9": 0.5920151074727377 + }, + "matched_cos_signed_band": 0.5920151074727377, + "matched_cos_signed_band_std": 0.003127111931646074, + "unique_frac_signed_per_threshold": { + "0.5": 0.9998914930555557, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 1.0, + "unique_frac_signed_band_std": 0.0, + "matched_frac_signed_per_threshold": { + "0.5": 0.6901041666666666, + "0.6": 0.5926649305555555, + "0.7": 0.4134114583333333, + "0.8": 0.20301649305555555, + "0.9": 0.055338541666666664 + }, + "matched_frac_signed_band": 0.3161078559027778, + "matched_frac_signed_band_std": 0.0030246006650968714, + "babel_mu1_abs_mean": 0.3404751221338908, + "babel_mu1_abs_std": 0.008576422901457265, + "n_dup_abs_per_threshold": { + "0.5": 559.3333333333334, + "0.6": 528.6666666666666, + "0.7": 510.0, + "0.8": 496.6666666666667, + "0.9": 442.6666666666667 + }, + "n_dup_abs_band": 494.5, + "n_dup_abs_band_std": 42.09216079034195, + "n_dup_clusters_abs_per_threshold": { + "0.5": 279.6666666666667, + "0.6": 264.3333333333333, + "0.7": 255.0, + "0.8": 248.33333333333334, + "0.9": 221.33333333333334 + }, + "n_dup_clusters_abs_band": 247.25, + "n_dup_clusters_abs_band_std": 21.046080395170975, + "largest_dup_abs_per_threshold": { + "0.5": 2.0, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.0, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.595129668712616, + "dcs_abs_std": 0.0029682224550745743, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5602577604133533, + "0.6": 0.5634096073497656, + "0.7": 0.565292126867765, + "0.8": 0.5664428357364765, + "0.9": 0.5708819927272594 + }, + "dedup_dcs_abs_band": 0.5665066406703166, + "dedup_dcs_abs_band_std": 0.005351656345231775, + "matched_cos_abs_per_threshold": { + "0.5": 0.6163627505302429, + "0.6": 0.6164453625679016, + "0.7": 0.6164562503496805, + "0.8": 0.616252581278483, + "0.9": 0.6151984333992004 + }, + "matched_cos_abs_band": 0.6160881568988164, + "matched_cos_abs_band_std": 0.0038122884831216152, + "unique_frac_abs_per_threshold": { + "0.5": 0.9089626736111112, + "0.6": 0.9139539930555555, + "0.7": 0.9169921875, + "0.8": 0.9191623263888888, + "0.9": 0.9279513888888888 + }, + "unique_frac_abs_band": 0.9195149739583333, + "unique_frac_abs_band_std": 0.0034254688143180974, + "matched_frac_abs_per_threshold": { + "0.5": 0.664279513888889, + "0.6": 0.5747612847222222, + "0.7": 0.4014756944444445, + "0.8": 0.19498697916666666, + "0.9": 0.053385416666666664 + }, + "matched_frac_abs_band": 0.30615234375000006, + "matched_frac_abs_band_std": 0.0032582586619738785, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/config.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..302eee48c766f1dcdb2d85c3c4cecf838b66a3d7 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3349256d77fa051908cf2ad877ea1c27c90c07a2 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:674554a0956261432a1a6d38dad260fd2f4b3fa2bd596f103ae0205273fbcbb0 +size 18890116 diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..09490dd6d8e3c31d09bb8f91cc8c598b54660474 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:241928d36e6fcd62a052fa72131391aa0f645896d00bd42fcef3c001926c06c0 +size 25920 diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2231239f6e8f0057c3b19cd2bfd6f8d6e4c356e --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4966c8a9858d95d12310c6fcdcb559c835590b3d7613c360a2979b890f87fef +size 13607 diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..ef973d4f18c4da0973dcb42c61f0e188c089c25e --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.30693959960936, + "l0_per_token_0.001": 128.30693959960936, + "l0_per_token_0.01": 128.30693959960936, + "l0_per_token_0.05": 128.30693959960936, + "l0_per_token_0.1": 128.30693959960936, + "l0_per_token_0.5": 128.30693959960936, + "l0_per_token_1.0": 128.30693959960936, + "l1_per_token": 810.0086997059205, + "r2_score": 0.9309044060817548, + "l2_loss": 1.6204598446855711 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea84140a948cf18578863de75464284c7e70c07b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7460817b1d948f4b6caa720dcb4f1293ecfb4a99cfeb9e9fe5db221a78790e6 +size 13518 diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..804b3253218c386f189ce98dadfd99a0b15f1937 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44f549f3d73f964f55ac4581bad99a3158f96d262eda002182cc1953ac20854c +size 13523 diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0e2e7c1739d3867f5cb55f42ba3e7a3eb59091b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1886265f63185267da1adafd53158d7393e318c276c162e2ad48827ff53806fc +size 13523 diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed1/config.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..aadd6db1df53ffb0cbe4db859cda6475f2dea8fe --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f5a69b7fe9ca3ac4605e9680c062de0e5d4fede1 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a972c912e6d29834e516277fb9b35372594742c4022b4f85984e5830b26bde78 +size 18890116 diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..d0238cb848051c8c6eb6f803feed0655dd657abe --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.6742666015625, + "l0_per_token_0.001": 127.6742666015625, + "l0_per_token_0.01": 127.6742666015625, + "l0_per_token_0.05": 127.6742666015625, + "l0_per_token_0.1": 127.6742666015625, + "l0_per_token_0.5": 127.6742666015625, + "l0_per_token_1.0": 127.6742666015625, + "l1_per_token": 809.9053929874115, + "r2_score": 0.9305413569658916, + "l2_loss": 1.6289742300543582 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/config.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f453baeacffb08d7c2804e003c32605c9dd3ba1b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..cb7610f8885273a7f21920636b8ea6d8b346911e --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.89192016601562, + "l0_per_token_0.001": 127.89192016601562, + "l0_per_token_0.01": 127.89192016601562, + "l0_per_token_0.05": 127.89192016601562, + "l0_per_token_0.1": 127.89192016601562, + "l0_per_token_0.5": 127.89192016601562, + "l0_per_token_1.0": 127.89192016601562, + "l1_per_token": 818.6819381570849, + "r2_score": 0.9307402745408487, + "l2_loss": 1.6243091287889755 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba8dde3e7e56ce30be2f8844bdc35ab43fb74b74 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b35f67c3389e757d573aee9f6227a629f596f36ea08962b594c717ca71b7f4f2 +size 13518 diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fd28e569ed695fecf1f35e66f89118f8245a13f --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9084d75e5d1b8eb46e2d1134bdb9ffe61b6e6ea7af8a3c2437884016abf384d2 +size 13523 diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..51dff20dbb18496b4d94ca63b921714279c2fda8 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4596549b93a9ddc43eafa4c1faffbbbab806a2c5a07fb04571a3cdc522b723ac +size 13523 diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_32/code_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..790e28311685c344edaf237cbeeaf27136f952ac --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 32.01258833333333, + "recon_r2": 0.8839271017054231, + "l1_per_token": 373.77083450625656 + }, + "seed1": { + "l0": 31.921448333333334, + "recon_r2": 0.8836812814286432, + "l1_per_token": 371.92725643414656 + }, + "seed2": { + "l0": 31.959998333333335, + "recon_r2": 0.8837495167241791, + "l1_per_token": 371.11894949056705 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6865253092747584, + "mcc_abs": 0.6865740183841833, + "act_iou": 0.36357290697005357, + "act_iou_n_pairs": 3072, + "input_r2": 0.9445742744338751, + "matched_mcc_signed": { + "0.6": 0.8136716929834653, + "0.7": 0.8405631469016986, + "0.8": 0.8736024601914064, + "0.9": 0.9154176923008508 + }, + "matched_mcc_abs": { + "0.6": 0.8119412158118958, + "0.7": 0.8397377759865362, + "0.8": 0.872728867117768, + "0.9": 0.9152071175526709 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6905472299818841, + "mcc_abs": 0.6905910750304995, + "act_iou": 0.369598778754795, + "act_iou_n_pairs": 3072, + "input_r2": 0.944788821584087, + "matched_mcc_signed": { + "0.6": 0.8141163427182354, + "0.7": 0.8408588695708695, + "0.8": 0.8738898561514108, + "0.9": 0.9153903819234055 + }, + "matched_mcc_abs": { + "0.6": 0.8124413783996326, + "0.7": 0.8398635312763558, + "0.8": 0.8731199966087836, + "0.9": 0.9138998648274427 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6929720876672336, + "mcc_abs": 0.693017937648024, + "act_iou": 0.3701589948934758, + "act_iou_n_pairs": 3072, + "input_r2": 0.9455289307706353, + "matched_mcc_signed": { + "0.6": 0.8157001904211801, + "0.7": 0.8407926826019945, + "0.8": 0.8750998050821889, + "0.9": 0.9147687311650495 + }, + "matched_mcc_abs": { + "0.6": 0.8131252096694283, + "0.7": 0.8399026597560804, + "0.8": 0.8728312002347771, + "0.9": 0.9134575023338952 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 31.964678333333335, + "l0_std": 0.045749881967060466, + "recon_r2_mean": 0.8837859666194152, + "recon_r2_std": 0.00012689896873526113, + "l1_per_token_mean": 372.27234681032337, + "l1_per_token_std": 1.3592053045463905, + "input_r2_mean": 0.9449640089295324, + "input_r2_std": 0.000500859396270317, + "mcc_signed_mean": 0.690014875641292, + "mcc_signed_std": 0.003256192372754059, + "mcc_abs_mean": 0.6900610103542356, + "mcc_abs_std": 0.0032544969335507722, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8144960753742936, + "0.7": 0.8407382330248542, + "0.8": 0.8741973738083354, + "0.9": 0.9151922684631019 + }, + "matched_mcc_signed_band": 0.8611559876676462, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8125026012936521, + "0.7": 0.8398346556729908, + "0.8": 0.8728933546537764, + "0.9": 0.9141881615713362 + }, + "matched_mcc_abs_band": 0.8598546932979388, + "act_iou_mean": 0.3677768935394415, + "act_iou_std": 0.003651518565912679, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_32/dict_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..d8d61d35715d49f6f8976cb435e5b32ffcaed38a --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 32, + "l0_raw": 32.0277763671875, + "recon_r2": 0.8838007775541042, + "l1_per_token": 373.5708489316518, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 27, + "dense_rate": 0.0087890625, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.19259123504161835, + "n_dup_abs": { + "0.5": 102, + "0.6": 62, + "0.7": 40, + "0.8": 26, + "0.9": 10 + }, + "n_dup_clusters_abs": { + "0.5": 51, + "0.6": 31, + "0.7": 20, + "0.8": 13, + "0.9": 5 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.25680527091026306 + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 32, + "l0_raw": 31.922845654296875, + "recon_r2": 0.8835723056471638, + "l1_per_token": 371.6550968160335, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 26, + "dense_rate": 0.008463541666666666, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.1912495344877243, + "n_dup_abs": { + "0.5": 90, + "0.6": 46, + "0.7": 38, + "0.8": 20, + "0.9": 8 + }, + "n_dup_clusters_abs": { + "0.5": 45, + "0.6": 23, + "0.7": 19, + "0.8": 10, + "0.9": 4 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2560412585735321 + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 32, + "l0_raw": 31.9730177734375, + "recon_r2": 0.8836215099479398, + "l1_per_token": 370.9001996175371, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 26, + "dense_rate": 0.008463541666666666, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.19113075733184814, + "n_dup_abs": { + "0.5": 103, + "0.6": 60, + "0.7": 44, + "0.8": 34, + "0.9": 10 + }, + "n_dup_clusters_abs": { + "0.5": 51, + "0.6": 30, + "0.7": 22, + "0.8": 17, + "0.9": 5 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2606920897960663 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.7051088809967041, + "dcs_abs": 0.7087423205375671, + "frac_sign_flipped": 0.0458984375, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7051460146903992, + "0.6": 0.7051088213920593, + "0.7": 0.7051088213920593, + "0.8": 0.7051088213920593, + "0.9": 0.7051088213920593 + }, + "unique_frac_signed": { + "0.5": 0.9998372395833334, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7050312448312374, + "0.6": 0.7051088213920593, + "0.7": 0.7051088213920593, + "0.8": 0.7051088213920593, + "0.9": 0.7051088213920593 + }, + "matched_frac_signed": { + "0.5": 0.8219401041666666, + "0.6": 0.748046875, + "0.7": 0.6357421875, + "0.8": 0.4563802083333333, + "0.9": 0.18424479166666666 + }, + "matched_cos_abs": { + "0.5": 0.7094749808311462, + "0.6": 0.7092151641845703, + "0.7": 0.70830899477005, + "0.8": 0.7082422375679016, + "0.9": 0.7086420059204102 + }, + "unique_frac_abs": { + "0.5": 0.984375, + "0.6": 0.9912109375, + "0.7": 0.99365234375, + "0.8": 0.9962565104166666, + "0.9": 0.99853515625 + }, + "dedup_dcs_abs": { + "0.5": 0.6983894342556596, + "0.6": 0.7029818277806044, + "0.7": 0.7038128927524667, + "0.8": 0.7055909401290894, + "0.9": 0.7076039561070502 + }, + "matched_frac_abs": { + "0.5": 0.8108723958333334, + "0.6": 0.7438151041666666, + "0.7": 0.6328125, + "0.8": 0.4537760416666667, + "0.9": 0.18359375 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.7099398970603943, + "dcs_abs": 0.7133638858795166, + "frac_sign_flipped": 0.048177083333333336, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7099931240081787, + "0.6": 0.7099398970603943, + "0.7": 0.7099398970603943, + "0.8": 0.7099398970603943, + "0.9": 0.7099398970603943 + }, + "unique_frac_signed": { + "0.5": 0.9996744791666666, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7097620064547906, + "0.6": 0.7099398970603943, + "0.7": 0.7099398970603943, + "0.8": 0.7099398970603943, + "0.9": 0.7099398970603943 + }, + "matched_frac_signed": { + "0.5": 0.8284505208333334, + "0.6": 0.7581380208333334, + "0.7": 0.65234375, + "0.8": 0.4671223958333333, + "0.9": 0.19075520833333334 + }, + "matched_cos_abs": { + "0.5": 0.7131237983703613, + "0.6": 0.7134924530982971, + "0.7": 0.7133829593658447, + "0.8": 0.713371217250824, + "0.9": 0.7131729125976562 + }, + "unique_frac_abs": { + "0.5": 0.9832356770833334, + "0.6": 0.9900716145833334, + "0.7": 0.9931640625, + "0.8": 0.9951171875, + "0.9": 0.9983723958333334 + }, + "dedup_dcs_abs": { + "0.5": 0.7011687607349207, + "0.6": 0.7064086250320543, + "0.7": 0.7085063180420548, + "0.8": 0.7098879593540914, + "0.9": 0.7120121493935585 + }, + "matched_frac_abs": { + "0.5": 0.8170572916666666, + "0.6": 0.751953125, + "0.7": 0.6487630208333334, + "0.8": 0.4638671875, + "0.9": 0.1904296875 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.7105026841163635, + "dcs_abs": 0.7140093445777893, + "frac_sign_flipped": 0.048177083333333336, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7105557918548584, + "0.6": 0.7105028033256531, + "0.7": 0.7105028033256531, + "0.8": 0.7105028033256531, + "0.9": 0.7105028033256531 + }, + "unique_frac_signed": { + "0.5": 0.99951171875, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7102088407846168, + "0.6": 0.7105028033256531, + "0.7": 0.7105028033256531, + "0.8": 0.7105028033256531, + "0.9": 0.7105028033256531 + }, + "matched_frac_signed": { + "0.5": 0.826171875, + "0.6": 0.7600911458333334, + "0.7": 0.6533203125, + "0.8": 0.46875, + "0.9": 0.2001953125 + }, + "matched_cos_abs": { + "0.5": 0.7147259712219238, + "0.6": 0.7144449353218079, + "0.7": 0.7138947248458862, + "0.8": 0.7139176726341248, + "0.9": 0.7137495279312134 + }, + "unique_frac_abs": { + "0.5": 0.9842122395833334, + "0.6": 0.9913736979166666, + "0.7": 0.9933268229166666, + "0.8": 0.99560546875, + "0.9": 0.99853515625 + }, + "dedup_dcs_abs": { + "0.5": 0.7034420488247027, + "0.6": 0.7082819174878143, + "0.7": 0.709130778928132, + "0.8": 0.7107803391118068, + "0.9": 0.7127039963961579 + }, + "matched_frac_abs": { + "0.5": 0.8186848958333334, + "0.6": 0.7555338541666666, + "0.7": 0.6487630208333334, + "0.8": 0.4664713541666667, + "0.9": 0.19921875 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 26, + "n_dense_std": 0.5773502691896258, + "l0_mean": 32, + "l0_std": 0.0524820598494907, + "recon_r2_mean": 0.8836648643830692, + "recon_r2_std": null, + "l1_per_token_mean": 372.0420484550741, + "l1_per_token_std": 1.376731852530003, + "babel_mu1_signed_mean": 0.19165717562039694, + "babel_mu1_signed_std": 0.0008110963290318058, + "n_dup_signed_per_threshold": { + "0.5": 2.0, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.0, + "n_dup_signed_band_std": 0.0, + "n_dup_clusters_signed_per_threshold": { + "0.5": 1.0, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.0, + "n_dup_clusters_signed_band_std": 0.0, + "largest_dup_signed_per_threshold": { + "0.5": 1.6666666666666667, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.0, + "largest_dup_signed_band_std": 0.0, + "dcs_signed_mean": 0.7085171540578207, + "dcs_signed_std": 0.002965033939424833, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7083340306902149, + "0.6": 0.7085171739260355, + "0.7": 0.7085171739260355, + "0.8": 0.7085171739260355, + "0.9": 0.7085171739260355 + }, + "dedup_dcs_signed_band": 0.7085171739260355, + "dedup_dcs_signed_band_std": 0.0029651081114798455, + "matched_cos_signed_per_threshold": { + "0.5": 0.7085649768511454, + "0.6": 0.7085171739260355, + "0.7": 0.7085171739260355, + "0.8": 0.7085171739260355, + "0.9": 0.7085171739260355 + }, + "matched_cos_signed_band": 0.7085171739260355, + "matched_cos_signed_band_std": 0.0029651081114798455, + "unique_frac_signed_per_threshold": { + "0.5": 0.9996744791666666, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 1.0, + "unique_frac_signed_band_std": 0.0, + "matched_frac_signed_per_threshold": { + "0.5": 0.8255208333333334, + "0.6": 0.7554253472222223, + "0.7": 0.6471354166666666, + "0.8": 0.46408420138888884, + "0.9": 0.19173177083333334 + }, + "matched_frac_signed_band": 0.5145941840277778, + "matched_frac_signed_band_std": 0.007558435567463288, + "babel_mu1_abs_mean": 0.2578462064266205, + "babel_mu1_abs_std": 0.00249403645262763, + "n_dup_abs_per_threshold": { + "0.5": 98.33333333333333, + "0.6": 56.0, + "0.7": 40.666666666666664, + "0.8": 26.666666666666668, + "0.9": 9.333333333333334 + }, + "n_dup_abs_band": 33.166666666666664, + "n_dup_abs_band_std": 4.645786621588785, + "n_dup_clusters_abs_per_threshold": { + "0.5": 49.0, + "0.6": 28.0, + "0.7": 20.333333333333332, + "0.8": 13.333333333333334, + "0.9": 4.666666666666667 + }, + "n_dup_clusters_abs_band": 16.583333333333332, + "n_dup_clusters_abs_band_std": 2.3228933107943925, + "largest_dup_abs_per_threshold": { + "0.5": 2.3333333333333335, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.0, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.712038516998291, + "dcs_abs_std": 0.002872775237215824, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.701000081271761, + "0.6": 0.7058907901001575, + "0.7": 0.7071499965742177, + "0.8": 0.7087530795316624, + "0.9": 0.7107733672989222 + }, + "dedup_dcs_abs_band": 0.70814180837624, + "dedup_dcs_abs_band_std": 0.002770525361417825, + "matched_cos_abs_per_threshold": { + "0.5": 0.7124415834744772, + "0.6": 0.7123841842015585, + "0.7": 0.7118622263272604, + "0.8": 0.7118437091509501, + "0.9": 0.7118548154830933 + }, + "matched_cos_abs_band": 0.7119862337907156, + "matched_cos_abs_band_std": 0.0029485361061725433, + "unique_frac_abs_per_threshold": { + "0.5": 0.9839409722222223, + "0.6": 0.9908854166666666, + "0.7": 0.9933810763888888, + "0.8": 0.9956597222222222, + "0.9": 0.9984809027777778 + }, + "unique_frac_abs_band": 0.9946017795138888, + "unique_frac_abs_band_std": 0.00037807508313704105, + "matched_frac_abs_per_threshold": { + "0.5": 0.8155381944444445, + "0.6": 0.7504340277777777, + "0.7": 0.6434461805555557, + "0.8": 0.46137152777777785, + "0.9": 0.19108072916666666 + }, + "matched_frac_abs_band": 0.5115831163194445, + "matched_frac_abs_band_std": 0.007246647474974644, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed0/config.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3373894456a1d6c6b080169c1b85162d45a7592b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..5d70c9b1ea8b49a7922b251b9543bbb5e241df20 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.0277763671875, + "l0_per_token_0.001": 32.0277763671875, + "l0_per_token_0.01": 32.0277763671875, + "l0_per_token_0.05": 32.0277763671875, + "l0_per_token_0.1": 32.0277763671875, + "l0_per_token_0.5": 32.0277763671875, + "l0_per_token_1.0": 32.0277763671875, + "l1_per_token": 373.5708489316518, + "r2_score": 0.8838007775541042, + "l2_loss": 2.7251545761377365 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed1/config.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bcc064d33eaf88f78bf489590c54dc9524bc6143 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..9640fa02ca81f84ed2674a92b4d7521b033f5cb6 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.922845654296875, + "l0_per_token_0.001": 31.922845654296875, + "l0_per_token_0.01": 31.922845654296875, + "l0_per_token_0.05": 31.922845654296875, + "l0_per_token_0.1": 31.922845654296875, + "l0_per_token_0.5": 31.922845654296875, + "l0_per_token_1.0": 31.922845654296875, + "l1_per_token": 371.6550968160335, + "r2_score": 0.8835723056471638, + "l2_loss": 2.7305127984185056 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed2/config.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..82e88bb0886f7281a92deafc94ccfbf0c2d948b3 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..997b17a85fabf615c3e7e1425060f1cc79b3cd7d --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.9730177734375, + "l0_per_token_0.001": 31.9730177734375, + "l0_per_token_0.01": 31.9730177734375, + "l0_per_token_0.05": 31.9730177734375, + "l0_per_token_0.1": 31.9730177734375, + "l0_per_token_0.5": 31.9730177734375, + "l0_per_token_1.0": 31.9730177734375, + "l1_per_token": 370.9001996175371, + "r2_score": 0.8836215099479398, + "l2_loss": 2.7293588378101394 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_64/code_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..0cba9ce4952b16dca5bf7ae8922a216bbce9474d --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 64.08201833333334, + "recon_r2": 0.906649419909342, + "l1_per_token": 533.7070892482249 + }, + "seed1": { + "l0": 63.897415, + "recon_r2": 0.9065400435305827, + "l1_per_token": 535.2668953068033 + }, + "seed2": { + "l0": 63.965756666666664, + "recon_r2": 0.9063842332183667, + "l1_per_token": 535.4323285220361 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6735134974228796, + "mcc_abs": 0.6735875917087567, + "act_iou": 0.331270438855823, + "act_iou_n_pairs": 3072, + "input_r2": 0.9427153811331435, + "matched_mcc_signed": { + "0.6": 0.812979811097555, + "0.7": 0.8426594320409925, + "0.8": 0.8794857956274637, + "0.9": 0.9269689104189349 + }, + "matched_mcc_abs": { + "0.6": 0.8101418896707397, + "0.7": 0.8409073846684201, + "0.8": 0.878706498600328, + "0.9": 0.9273501009547268 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6688453172532544, + "mcc_abs": 0.6689231460102428, + "act_iou": 0.32513826772849463, + "act_iou_n_pairs": 3072, + "input_r2": 0.9413365888400912, + "matched_mcc_signed": { + "0.6": 0.8092382152455526, + "0.7": 0.8392192820440166, + "0.8": 0.8773999731743927, + "0.9": 0.9274745840002823 + }, + "matched_mcc_abs": { + "0.6": 0.8076386530789575, + "0.7": 0.8383108996511701, + "0.8": 0.8780158392606942, + "0.9": 0.9268801311644109 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6690732636289151, + "mcc_abs": 0.6691373146954952, + "act_iou": 0.32384264678702374, + "act_iou_n_pairs": 3072, + "input_r2": 0.9416405298478424, + "matched_mcc_signed": { + "0.6": 0.8074541392935748, + "0.7": 0.8374203647240723, + "0.8": 0.8758658862534733, + "0.9": 0.9253427145912855 + }, + "matched_mcc_abs": { + "0.6": 0.8054204129379205, + "0.7": 0.8370821089296874, + "0.8": 0.8750499912413178, + "0.9": 0.92588689362355 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 63.98173, + "l0_std": 0.09333251417795435, + "recon_r2_mean": 0.9065245655527638, + "recon_r2_std": 0.00013326916791077333, + "l1_per_token_mean": 534.8021043590214, + "l1_per_token_std": 0.951911554100575, + "input_r2_mean": 0.941897499940359, + "input_r2_std": 0.0007244254745528198, + "mcc_signed_mean": 0.6704773594350164, + "mcc_signed_std": 0.0026318416173003144, + "mcc_abs_mean": 0.6705493508048316, + "mcc_abs_std": 0.002633371963689997, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8098907218788941, + "0.7": 0.839766359603027, + "0.8": 0.8775838850184433, + "0.9": 0.926595403003501 + }, + "matched_mcc_signed_band": 0.8634590923759664, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8077336518958727, + "0.7": 0.8387667977497593, + "0.8": 0.8772574430341132, + "0.9": 0.926705708580896 + }, + "matched_mcc_abs_band": 0.8626159003151602, + "act_iou_mean": 0.3267504511237804, + "act_iou_std": 0.003967666219499306, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_64/dict_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..925393b96bfe62e5b5a4ab82923a5d2d59c87244 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 64, + "l0_raw": 64.09867075195312, + "recon_r2": 0.906551280194744, + "l1_per_token": 533.4875236956665, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 134, + "dense_rate": 0.043619791666666664, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.15983740985393524, + "n_dup_abs": { + "0.5": 170, + "0.6": 142, + "0.7": 116, + "0.8": 86, + "0.9": 24 + }, + "n_dup_clusters_abs": { + "0.5": 85, + "0.6": 71, + "0.7": 58, + "0.8": 43, + "0.9": 12 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.255954384803772 + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 64, + "l0_raw": 63.87882587890625, + "recon_r2": 0.9064355754375961, + "l1_per_token": 534.82485491274, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 133, + "dense_rate": 0.043294270833333336, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.15749551355838776, + "n_dup_abs": { + "0.5": 162, + "0.6": 116, + "0.7": 102, + "0.8": 84, + "0.9": 30 + }, + "n_dup_clusters_abs": { + "0.5": 81, + "0.6": 58, + "0.7": 51, + "0.8": 42, + "0.9": 15 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.25281527638435364 + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 64, + "l0_raw": 63.952916162109375, + "recon_r2": 0.9062618956824577, + "l1_per_token": 534.9672415860614, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 132, + "dense_rate": 0.04296875, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.15847934782505035, + "n_dup_abs": { + "0.5": 195, + "0.6": 145, + "0.7": 123, + "0.8": 109, + "0.9": 42 + }, + "n_dup_clusters_abs": { + "0.5": 97, + "0.6": 72, + "0.7": 61, + "0.8": 54, + "0.9": 21 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 3, + "0.9": 2 + }, + "babel_mu1_abs": 0.25697293877601624 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.6712257266044617, + "dcs_abs": 0.6743609309196472, + "frac_sign_flipped": 0.048177083333333336, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.6713180541992188, + "0.6": 0.6712257266044617, + "0.7": 0.6712257266044617, + "0.8": 0.6712257266044617, + "0.9": 0.6712257266044617 + }, + "unique_frac_signed": { + "0.5": 0.9998372395833334, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6712087901930015, + "0.6": 0.6712257266044617, + "0.7": 0.6712257266044617, + "0.8": 0.6712257266044617, + "0.9": 0.6712257266044617 + }, + "matched_frac_signed": { + "0.5": 0.787109375, + "0.6": 0.703125, + "0.7": 0.5693359375, + "0.8": 0.3623046875, + "0.9": 0.12825520833333334 + }, + "matched_cos_abs": { + "0.5": 0.6774513125419617, + "0.6": 0.6781340837478638, + "0.7": 0.6773392558097839, + "0.8": 0.6756635904312134, + "0.9": 0.674886167049408 + }, + "unique_frac_abs": { + "0.5": 0.9729817708333334, + "0.6": 0.97900390625, + "0.7": 0.9822591145833334, + "0.8": 0.9861653645833334, + "0.9": 0.99560546875 + }, + "dedup_dcs_abs": { + "0.5": 0.6591477777304439, + "0.6": 0.6638959169504233, + "0.7": 0.6653226576842524, + "0.8": 0.6663160309932816, + "0.9": 0.6719203586981166 + }, + "matched_frac_abs": { + "0.5": 0.7737630208333334, + "0.6": 0.6988932291666666, + "0.7": 0.5654296875, + "0.8": 0.359375, + "0.9": 0.126953125 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.6653919816017151, + "dcs_abs": 0.6686840653419495, + "frac_sign_flipped": 0.052083333333333336, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.6653845310211182, + "0.6": 0.6655586957931519, + "0.7": 0.6655586957931519, + "0.8": 0.6655586957931519, + "0.9": 0.6653919816017151 + }, + "unique_frac_signed": { + "0.5": 0.9996744791666666, + "0.6": 0.9998372395833334, + "0.7": 0.9998372395833334, + "0.8": 0.9998372395833334, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6651679344940931, + "0.6": 0.6654503691825084, + "0.7": 0.6654503691825084, + "0.8": 0.6654503691825084, + "0.9": 0.6653919816017151 + }, + "matched_frac_signed": { + "0.5": 0.7864583333333334, + "0.6": 0.697265625, + "0.7": 0.5608723958333334, + "0.8": 0.3421223958333333, + "0.9": 0.1123046875 + }, + "matched_cos_abs": { + "0.5": 0.6718516945838928, + "0.6": 0.6711485385894775, + "0.7": 0.6710292100906372, + "0.8": 0.6711476445198059, + "0.9": 0.669011652469635 + }, + "unique_frac_abs": { + "0.5": 0.97021484375, + "0.6": 0.9765625, + "0.7": 0.98046875, + "0.8": 0.9840494791666666, + "0.9": 0.99462890625 + }, + "dedup_dcs_abs": { + "0.5": 0.6518404868838843, + "0.6": 0.6554184947162867, + "0.7": 0.6579231708310544, + "0.8": 0.6604424900336501, + "0.9": 0.6654183281643782 + }, + "matched_frac_abs": { + "0.5": 0.76953125, + "0.6": 0.6881510416666666, + "0.7": 0.5543619791666666, + "0.8": 0.3369140625, + "0.9": 0.1103515625 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.6650788187980652, + "dcs_abs": 0.6682863235473633, + "frac_sign_flipped": 0.05078125, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.6652875542640686, + "0.6": 0.6652875542640686, + "0.7": 0.6652875542640686, + "0.8": 0.6652875542640686, + "0.9": 0.6650787591934204 + }, + "unique_frac_signed": { + "0.5": 0.9998372395833334, + "0.6": 0.9998372395833334, + "0.7": 0.9998372395833334, + "0.8": 0.9998372395833334, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6651792717845335, + "0.6": 0.6651792717845335, + "0.7": 0.6651792717845335, + "0.8": 0.6651792717845335, + "0.9": 0.6650787591934204 + }, + "matched_frac_signed": { + "0.5": 0.7890625, + "0.6": 0.6966145833333334, + "0.7": 0.5563151041666666, + "0.8": 0.341796875, + "0.9": 0.10970052083333333 + }, + "matched_cos_abs": { + "0.5": 0.672355055809021, + "0.6": 0.6727349162101746, + "0.7": 0.672143816947937, + "0.8": 0.6715680360794067, + "0.9": 0.6691527962684631 + }, + "unique_frac_abs": { + "0.5": 0.9708658854166666, + "0.6": 0.9786783854166666, + "0.7": 0.9816080729166666, + "0.8": 0.9842122395833334, + "0.9": 0.994140625 + }, + "dedup_dcs_abs": { + "0.5": 0.6527665865723975, + "0.6": 0.6583911216099901, + "0.7": 0.6597817968771171, + "0.8": 0.6609654808222938, + "0.9": 0.6652319791028276 + }, + "matched_frac_abs": { + "0.5": 0.7721354166666666, + "0.6": 0.6901041666666666, + "0.7": 0.5501302083333334, + "0.8": 0.337890625, + "0.9": 0.1083984375 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 133, + "n_dense_std": 1.0, + "l0_mean": 64, + "l0_std": 0.11185223789344785, + "recon_r2_mean": 0.9064162504382659, + "recon_r2_std": null, + "l1_per_token_mean": 534.4265400648227, + "l1_per_token_std": 0.8163224109322645, + "babel_mu1_signed_mean": 0.1586040904124578, + "babel_mu1_signed_std": 0.0011759209580620561, + "n_dup_signed_per_threshold": { + "0.5": 1.3333333333333333, + "0.6": 0.6666666666666666, + "0.7": 0.6666666666666666, + "0.8": 0.6666666666666666, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.5, + "n_dup_signed_band_std": 0.8660254037844386, + "n_dup_clusters_signed_per_threshold": { + "0.5": 0.6666666666666666, + "0.6": 0.3333333333333333, + "0.7": 0.3333333333333333, + "0.8": 0.3333333333333333, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.25, + "n_dup_clusters_signed_band_std": 0.4330127018922193, + "largest_dup_signed_per_threshold": { + "0.5": 1.6666666666666667, + "0.6": 1.3333333333333333, + "0.7": 1.3333333333333333, + "0.8": 1.3333333333333333, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.25, + "largest_dup_signed_band_std": 0.4330127018922193, + "dcs_signed_mean": 0.6672321756680807, + "dcs_signed_std": 0.0034620592927475653, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.6671853321572093, + "0.6": 0.6672851225238347, + "0.7": 0.6672851225238347, + "0.8": 0.6672851225238347, + "0.9": 0.6672321557998657 + }, + "dedup_dcs_signed_band": 0.6672718808428424, + "dedup_dcs_signed_band_std": 0.003427025080767551, + "matched_cos_signed_per_threshold": { + "0.5": 0.6673300464948019, + "0.6": 0.667357325553894, + "0.7": 0.667357325553894, + "0.8": 0.667357325553894, + "0.9": 0.6672321557998657 + }, + "matched_cos_signed_band": 0.667326033115387, + "matched_cos_signed_band_std": 0.0033801686816999446, + "unique_frac_signed_per_threshold": { + "0.5": 0.9997829861111112, + "0.6": 0.9998914930555557, + "0.7": 0.9998914930555557, + "0.8": 0.9998914930555557, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9999186197916667, + "unique_frac_signed_band_std": 7.047732778193673e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.7875434027777778, + "0.6": 0.6990017361111112, + "0.7": 0.5621744791666666, + "0.8": 0.3487413194444444, + "0.9": 0.11675347222222222 + }, + "matched_frac_signed_band": 0.4316677517361111, + "matched_frac_signed_band_std": 0.007935439711539746, + "babel_mu1_abs_mean": 0.2552475333213806, + "babel_mu1_abs_std": 0.0021670875396209844, + "n_dup_abs_per_threshold": { + "0.5": 175.66666666666666, + "0.6": 134.33333333333334, + "0.7": 113.66666666666667, + "0.8": 93.0, + "0.9": 32.0 + }, + "n_dup_abs_band": 93.25, + "n_dup_abs_band_std": 10.928746497197197, + "n_dup_clusters_abs_per_threshold": { + "0.5": 87.66666666666667, + "0.6": 67.0, + "0.7": 56.666666666666664, + "0.8": 46.333333333333336, + "0.9": 16.0 + }, + "n_dup_clusters_abs_band": 46.5, + "n_dup_clusters_abs_band_std": 5.267826876426369, + "largest_dup_abs_per_threshold": { + "0.5": 2.3333333333333335, + "0.6": 2.3333333333333335, + "0.7": 2.3333333333333335, + "0.8": 2.3333333333333335, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.25, + "largest_dup_abs_band_std": 0.4330127018922193, + "dcs_abs_mean": 0.6704437732696533, + "dcs_abs_std": 0.003398182260410621, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6545849503955753, + "0.6": 0.6592351777589, + "0.7": 0.6610092084641414, + "0.8": 0.6625746672830751, + "0.9": 0.6675235553217741 + }, + "dedup_dcs_abs_band": 0.6625856522069726, + "dedup_dcs_abs_band_std": 0.0037608286723481094, + "matched_cos_abs_per_threshold": { + "0.5": 0.6738860209782919, + "0.6": 0.6740058461825053, + "0.7": 0.673504094282786, + "0.8": 0.6727930903434753, + "0.9": 0.6710168719291687 + }, + "matched_cos_abs_band": 0.6728299756844838, + "matched_cos_abs_band_std": 0.0032093510918855452, + "unique_frac_abs_per_threshold": { + "0.5": 0.9713541666666666, + "0.6": 0.9780815972222222, + "0.7": 0.9814453125, + "0.8": 0.9848090277777778, + "0.9": 0.9947916666666666 + }, + "unique_frac_abs_band": 0.9847819010416666, + "unique_frac_abs_band_std": 0.0009216106488368202, + "matched_frac_abs_per_threshold": { + "0.5": 0.7718098958333334, + "0.6": 0.6923828125, + "0.7": 0.556640625, + "0.8": 0.3447265625, + "0.9": 0.115234375 + }, + "matched_frac_abs_band": 0.42724609375, + "matched_frac_abs_band_std": 0.0090302700286892, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed0/config.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7802b1620e053d9a724ed6ee8b685a6f429fd36c --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..a24ad09b70d2d03c727080d068b4a352850f878a --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.09867075195312, + "l0_per_token_0.001": 64.09867075195312, + "l0_per_token_0.01": 64.09867075195312, + "l0_per_token_0.05": 64.09867075195312, + "l0_per_token_0.1": 64.09867075195312, + "l0_per_token_0.5": 64.09867075195312, + "l0_per_token_1.0": 64.09867075195312, + "l1_per_token": 533.4875236956665, + "r2_score": 0.906551280194744, + "l2_loss": 2.1915999182359553 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed1/config.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f5667ba42ee29ab3985e17f4dc8dacd59a854842 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..ecdc1c4ff5e1167497da180056548bda1ddfef1a --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.87882587890625, + "l0_per_token_0.001": 63.87882587890625, + "l0_per_token_0.01": 63.87882587890625, + "l0_per_token_0.05": 63.87882587890625, + "l0_per_token_0.1": 63.87882587890625, + "l0_per_token_0.5": 63.87882587890625, + "l0_per_token_1.0": 63.87882587890625, + "l1_per_token": 534.82485491274, + "r2_score": 0.9064355754375961, + "l2_loss": 2.194313476397406 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed2/config.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a4eedd7c928a1af9fa774274161a7fc3fff23423 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..bbb707dfbeb4e60b54ef02371621c71259e7673e --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.952916162109375, + "l0_per_token_0.001": 63.952916162109375, + "l0_per_token_0.01": 63.952916162109375, + "l0_per_token_0.05": 63.952916162109375, + "l0_per_token_0.1": 63.952916162109375, + "l0_per_token_0.5": 63.952916162109375, + "l0_per_token_1.0": 63.952916162109375, + "l1_per_token": 534.9672415860614, + "r2_score": 0.9062618956824577, + "l2_loss": 2.1983866893633364 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/code_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..1b09f588884dcaf00d7bb3a9d0f7316bd1f91562 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 128.23526166666667, + "recon_r2": 0.9405176262357515, + "l1_per_token": 777.4512174000549 + }, + "seed1": { + "l0": 127.65849833333333, + "recon_r2": 0.9404339870483801, + "l1_per_token": 779.4682830249369 + }, + "seed2": { + "l0": 127.86450166666667, + "recon_r2": 0.9404511830459613, + "l1_per_token": 775.8662539034259 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6464015797726038, + "mcc_abs": 0.6464681970603617, + "act_iou": 0.3038581864876863, + "act_iou_n_pairs": 6144, + "input_r2": 0.9467310760092296, + "matched_mcc_signed": { + "0.6": 0.8184945639377775, + "0.7": 0.8479623517595947, + "0.8": 0.8760173242271985, + "0.9": 0.9243590978754762 + }, + "matched_mcc_abs": { + "0.6": 0.8237122181503352, + "0.7": 0.8539039363929797, + "0.8": 0.8867949245399755, + "0.9": 0.9337950158777455 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6462491437169228, + "mcc_abs": 0.6463066224182236, + "act_iou": 0.30286143983754465, + "act_iou_n_pairs": 6144, + "input_r2": 0.9468879216452295, + "matched_mcc_signed": { + "0.6": 0.8217281229555694, + "0.7": 0.8501409707017791, + "0.8": 0.8807553295806495, + "0.9": 0.9380777776274475 + }, + "matched_mcc_abs": { + "0.6": 0.8267142733135575, + "0.7": 0.8551254269341718, + "0.8": 0.8919956295980275, + "0.9": 0.9422677191027907 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6497473266200492, + "mcc_abs": 0.6498033144461793, + "act_iou": 0.3083157187482944, + "act_iou_n_pairs": 6144, + "input_r2": 0.9475163082284462, + "matched_mcc_signed": { + "0.6": 0.822216392843947, + "0.7": 0.8492059940890033, + "0.8": 0.87371459119336, + "0.9": 0.9271626390365518 + }, + "matched_mcc_abs": { + "0.6": 0.8272001765059249, + "0.7": 0.8548796523519101, + "0.8": 0.8835382283891735, + "0.9": 0.9341075393902066 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 127.91942055555556, + "l0_std": 0.2922773493396994, + "recon_r2_mean": 0.9404675987766976, + "recon_r2_std": 4.4169962392374184e-05, + "l1_per_token_mean": 777.5952514428059, + "l1_per_token_std": 1.805329001084494, + "input_r2_mean": 0.9470451019609684, + "input_r2_std": 0.00041554379825215393, + "mcc_signed_mean": 0.6474660167031919, + "mcc_signed_std": 0.0019771419752590805, + "mcc_abs_mean": 0.6475260446415882, + "mcc_abs_std": 0.0019738274781303435, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8208130265790979, + "0.7": 0.8491031055167925, + "0.8": 0.8768290816670694, + "0.9": 0.9298665048464918 + }, + "matched_mcc_signed_band": 0.8691529296523629, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8258755559899392, + "0.7": 0.8546363385596872, + "0.8": 0.8874429275090588, + "0.9": 0.9367234247902476 + }, + "matched_mcc_abs_band": 0.8761695617122333, + "act_iou_mean": 0.3050117816911751, + "act_iou_std": 0.0029043718820420724, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/dict_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..b3e741f99e9b78e01a7f2de079d6e843b64053b9 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 128, + "l0_raw": 128.25739013671875, + "recon_r2": 0.9404230882285594, + "l1_per_token": 776.998099812821, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 434, + "dense_rate": 0.07063802083333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.1674845963716507, + "n_dup_abs": { + "0.5": 864, + "0.6": 810, + "0.7": 776, + "0.8": 752, + "0.9": 662 + }, + "n_dup_clusters_abs": { + "0.5": 432, + "0.6": 405, + "0.7": 388, + "0.8": 376, + "0.9": 331 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.31925299763679504 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 128, + "l0_raw": 127.65566948242187, + "recon_r2": 0.9403530496789464, + "l1_per_token": 778.9610607068885, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 421, + "dense_rate": 0.06852213541666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.16720402240753174, + "n_dup_abs": { + "0.5": 892, + "0.6": 824, + "0.7": 810, + "0.8": 792, + "0.9": 706 + }, + "n_dup_clusters_abs": { + "0.5": 446, + "0.6": 412, + "0.7": 405, + "0.8": 396, + "0.9": 353 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.32232728600502014 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 128, + "l0_raw": 127.90002719726563, + "recon_r2": 0.9403705726506784, + "l1_per_token": 775.5630566501216, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 436, + "dense_rate": 0.07096354166666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.16703195869922638, + "n_dup_abs": { + "0.5": 847, + "0.6": 787, + "0.7": 749, + "0.8": 718, + "0.9": 646 + }, + "n_dup_clusters_abs": { + "0.5": 423, + "0.6": 393, + "0.7": 374, + "0.8": 359, + "0.9": 323 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.31630656123161316 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.6008937954902649, + "dcs_abs": 0.6040546298027039, + "frac_sign_flipped": 0.064453125, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6008937954902649, + "0.6": 0.6008937954902649, + "0.7": 0.6008937954902649, + "0.8": 0.6008937954902649, + "0.9": 0.6008937954902649 + }, + "unique_frac_signed": { + "0.5": 1.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6008937954902649, + "0.6": 0.6008937954902649, + "0.7": 0.6008937954902649, + "0.8": 0.6008937954902649, + "0.9": 0.6008937954902649 + }, + "matched_frac_signed": { + "0.5": 0.7288411458333334, + "0.6": 0.6105143229166666, + "0.7": 0.3982747395833333, + "0.8": 0.16292317708333334, + "0.9": 0.038736979166666664 + }, + "matched_cos_abs": { + "0.5": 0.613865852355957, + "0.6": 0.6138187050819397, + "0.7": 0.6141915917396545, + "0.8": 0.614206850528717, + "0.9": 0.6128490567207336 + }, + "unique_frac_abs": { + "0.5": 0.9285481770833334, + "0.6": 0.9335123697916666, + "0.7": 0.9354654947916666, + "0.8": 0.9371744791666666, + "0.9": 0.9443359375 + }, + "dedup_dcs_abs": { + "0.5": 0.5700040181788306, + "0.6": 0.5730073540034937, + "0.7": 0.5745550412636172, + "0.8": 0.575618985244849, + "0.9": 0.5787353885243647 + }, + "matched_frac_abs": { + "0.5": 0.7000325520833334, + "0.6": 0.5904947916666666, + "0.7": 0.3829752604166667, + "0.8": 0.15234375, + "0.9": 0.035970052083333336 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.6017178893089294, + "dcs_abs": 0.6050328016281128, + "frac_sign_flipped": 0.06396484375, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6018061637878418, + "0.6": 0.6018061637878418, + "0.7": 0.6018061637878418, + "0.8": 0.6017178893089294, + "0.9": 0.6017178893089294 + }, + "unique_frac_signed": { + "0.5": 0.9999186197916666, + "0.6": 0.9999186197916666, + "0.7": 0.9999186197916666, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6017571886768565, + "0.6": 0.6017571886768565, + "0.7": 0.6017571886768565, + "0.8": 0.6017178893089294, + "0.9": 0.6017178893089294 + }, + "matched_frac_signed": { + "0.5": 0.728515625, + "0.6": 0.6121419270833334, + "0.7": 0.4051106770833333, + "0.8": 0.1689453125, + "0.9": 0.037109375 + }, + "matched_cos_abs": { + "0.5": 0.614408016204834, + "0.6": 0.6142004728317261, + "0.7": 0.6138401627540588, + "0.8": 0.6139779686927795, + "0.9": 0.6128000020980835 + }, + "unique_frac_abs": { + "0.5": 0.9303385416666666, + "0.6": 0.9349772135416666, + "0.7": 0.9379069010416666, + "0.8": 0.940185546875, + "0.9": 0.94677734375 + }, + "dedup_dcs_abs": { + "0.5": 0.5716074577843149, + "0.6": 0.5742634466441814, + "0.7": 0.5757249247835716, + "0.8": 0.5772532122646226, + "0.9": 0.5801851582364179 + }, + "matched_frac_abs": { + "0.5": 0.6998697916666666, + "0.6": 0.59228515625, + "0.7": 0.3899739583333333, + "0.8": 0.15804036458333334, + "0.9": 0.03466796875 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.6052066683769226, + "dcs_abs": 0.6085711121559143, + "frac_sign_flipped": 0.0595703125, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6052933931350708, + "0.6": 0.6052933931350708, + "0.7": 0.6052933931350708, + "0.8": 0.6052066683769226, + "0.9": 0.6052066683769226 + }, + "unique_frac_signed": { + "0.5": 0.9999186197916666, + "0.6": 0.9999186197916666, + "0.7": 0.9999186197916666, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6052441342326347, + "0.6": 0.6052441342326347, + "0.7": 0.6052441342326347, + "0.8": 0.6052066683769226, + "0.9": 0.6052066683769226 + }, + "matched_frac_signed": { + "0.5": 0.73388671875, + "0.6": 0.6194661458333334, + "0.7": 0.4163411458333333, + "0.8": 0.17415364583333334, + "0.9": 0.041341145833333336 + }, + "matched_cos_abs": { + "0.5": 0.6180564165115356, + "0.6": 0.6184121370315552, + "0.7": 0.6188734769821167, + "0.8": 0.6188848614692688, + "0.9": 0.617226243019104 + }, + "unique_frac_abs": { + "0.5": 0.92919921875, + "0.6": 0.9344075520833334, + "0.7": 0.9365234375, + "0.8": 0.9385579427083334, + "0.9": 0.9449869791666666 + }, + "dedup_dcs_abs": { + "0.5": 0.5742975393659435, + "0.6": 0.5778489711422784, + "0.7": 0.5795895160408691, + "0.8": 0.5808593023539288, + "0.9": 0.5832707628530139 + }, + "matched_frac_abs": { + "0.5": 0.7052408854166666, + "0.6": 0.5986328125, + "0.7": 0.4002278645833333, + "0.8": 0.16259765625, + "0.9": 0.03759765625 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 430, + "n_dense_std": 8.144527815247079, + "l0_mean": 128, + "l0_std": 0.3026237237631664, + "recon_r2_mean": 0.9403822368527281, + "recon_r2_std": null, + "l1_per_token_mean": 777.1740723899437, + "l1_per_token_std": 1.7058232368652497, + "babel_mu1_signed_mean": 0.16724019249280295, + "babel_mu1_signed_std": 0.00022847630495484664, + "n_dup_signed_per_threshold": { + "0.5": 0.6666666666666666, + "0.6": 0.6666666666666666, + "0.7": 0.6666666666666666, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.3333333333333333, + "n_dup_signed_band_std": 0.5773502691896258, + "n_dup_clusters_signed_per_threshold": { + "0.5": 0.3333333333333333, + "0.6": 0.3333333333333333, + "0.7": 0.3333333333333333, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.16666666666666666, + "n_dup_clusters_signed_band_std": 0.2886751345948129, + "largest_dup_signed_per_threshold": { + "0.5": 1.3333333333333333, + "0.6": 1.3333333333333333, + "0.7": 1.3333333333333333, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.1666666666666665, + "largest_dup_signed_band_std": 0.2886751345948129, + "dcs_signed_mean": 0.6026061177253723, + "dcs_signed_std": 0.0022895262444333916, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.602631706133252, + "0.6": 0.602631706133252, + "0.7": 0.602631706133252, + "0.8": 0.6026061177253723, + "0.9": 0.6026061177253723 + }, + "dedup_dcs_signed_band": 0.6026189119293122, + "dedup_dcs_signed_band_std": 0.0022963701624240593, + "matched_cos_signed_per_threshold": { + "0.5": 0.6026644508043925, + "0.6": 0.6026644508043925, + "0.7": 0.6026644508043925, + "0.8": 0.6026061177253723, + "0.9": 0.6026061177253723 + }, + "matched_cos_signed_band": 0.6026352842648823, + "matched_cos_signed_band_std": 0.002305673589684208, + "unique_frac_signed_per_threshold": { + "0.5": 0.9999457465277777, + "0.6": 0.9999457465277777, + "0.7": 0.9999457465277777, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9999728732638888, + "unique_frac_signed_band_std": 2.3492442594021645e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.7304144965277778, + "0.6": 0.6140407986111112, + "0.7": 0.4065755208333333, + "0.8": 0.16867404513888892, + "0.9": 0.0390625 + }, + "matched_frac_signed_band": 0.3070882161458333, + "matched_frac_signed_band_std": 0.005222143302258008, + "babel_mu1_abs_mean": 0.31929561495780945, + "babel_mu1_abs_std": 0.0030105886262186293, + "n_dup_abs_per_threshold": { + "0.5": 867.6666666666666, + "0.6": 807.0, + "0.7": 778.3333333333334, + "0.8": 754.0, + "0.9": 671.3333333333334 + }, + "n_dup_abs_band": 752.6666666666667, + "n_dup_abs_band_std": 29.091808698211484, + "n_dup_clusters_abs_per_threshold": { + "0.5": 433.6666666666667, + "0.6": 403.3333333333333, + "0.7": 389.0, + "0.8": 377.0, + "0.9": 335.6666666666667 + }, + "n_dup_clusters_abs_band": 376.25, + "n_dup_clusters_abs_band_std": 14.665009376062464, + "largest_dup_abs_per_threshold": { + "0.5": 2.3333333333333335, + "0.6": 2.3333333333333335, + "0.7": 2.3333333333333335, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.166666666666667, + "largest_dup_abs_band_std": 0.2886751345948129, + "dcs_abs_mean": 0.605886181195577, + "dcs_abs_std": 0.0023760988460792996, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.571969671776363, + "0.6": 0.5750399239299845, + "0.7": 0.5766231606960193, + "0.8": 0.5779104999544667, + "0.9": 0.5807304365379321 + }, + "dedup_dcs_abs_band": 0.5775760052796006, + "dedup_dcs_abs_band_std": 0.0025342306388854577, + "matched_cos_abs_per_threshold": { + "0.5": 0.6154434283574423, + "0.6": 0.6154771049817404, + "0.7": 0.61563507715861, + "0.8": 0.6156898935635885, + "0.9": 0.614291767279307 + }, + "matched_cos_abs_band": 0.6152734607458115, + "matched_cos_abs_band_std": 0.0026638304853849834, + "unique_frac_abs_per_threshold": { + "0.5": 0.9293619791666666, + "0.6": 0.9342990451388888, + "0.7": 0.9366319444444443, + "0.8": 0.9386393229166666, + "0.9": 0.9453667534722222 + }, + "unique_frac_abs_band": 0.9387342664930555, + "unique_frac_abs_band_std": 0.0011740934307565012, + "matched_frac_abs_per_threshold": { + "0.5": 0.7017144097222222, + "0.6": 0.5938042534722222, + "0.7": 0.39105902777777773, + "0.8": 0.1576605902777778, + "0.9": 0.03607855902777778 + }, + "matched_frac_abs_band": 0.29465060763888884, + "matched_frac_abs_band_std": 0.004725018807623637, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/config.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..82407dded34aeb2c95a0024234ae0475abda7f80 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4357b3090e98c62f2fa8e60289940fe2be9d6d9d --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ff80524f6e061f7548e56c1e1cc8d4f7124739046ccdb50c4dd435c7843d341 +size 37776772 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7684911cf90b556e7ed0931a5836969ec3d74b36 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81a7041b8b2c2c1c51a987c23ad1d779c32f1b968c976d6e9531518d04d92f6d +size 50496 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..e88351236999071dea5638c5ea7223e1e8e2d30b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b2d17d35a41c5a6299d9b44cfdd12449c9943dec84150fc8a412f7963f7e0b2 +size 25895 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..f1b9738b59836acdf18b9b65fe7ababa867d2688 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.25739013671875, + "l0_per_token_0.001": 128.25739013671875, + "l0_per_token_0.01": 128.25739013671875, + "l0_per_token_0.05": 128.25739013671875, + "l0_per_token_0.1": 128.25739013671875, + "l0_per_token_0.5": 128.25739013671875, + "l0_per_token_1.0": 128.25739013671875, + "l1_per_token": 776.998099812821, + "r2_score": 0.9404230882285594, + "l2_loss": 1.3972235814374014 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7928d7db398c6473f238331f0ac2de4a5a7bd8ee --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112e306deaeb072487222db2ebb0dcd6dd0ff8671ecd950e50006bb8f3d2e6f2 +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0467dcf9bed750df5459ce32430f5e95cfd4d791 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:360d00a9e352cb7a5e0d2ca32a16e10e6f15f7275bdcff28dffaed76bf8913b7 +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/config.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c7a4a9fdcf08106174cb74ff78d2526490d56eb4 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ce04747abe26e5260bb45a03f95619daef4b6f0f --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68c2f71702ba867f091de46eb4ce65bed837d5fcdb275ff5f831d5f2029f7da1 +size 37776772 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b55f7ae94437b5aa82d919c89fe0afa464e6fc9 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51d4696397c4d2750346a89710ff0eb1694fe540c772d346b025490cc618067a +size 50496 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..370a39f66cad59cedfe1b58f0147980b47940170 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75f4473dfc767e510db712278dd24a37c1ab831f0e834bbbf1b76404d87feea4 +size 25895 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..4ab09cdbab64156266feb7d287ae720ff53fbda9 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.65566948242187, + "l0_per_token_0.001": 127.65566948242187, + "l0_per_token_0.01": 127.65566948242187, + "l0_per_token_0.05": 127.65566948242187, + "l0_per_token_0.1": 127.65566948242187, + "l0_per_token_0.5": 127.65566948242187, + "l0_per_token_1.0": 127.65566948242187, + "l1_per_token": 778.9610607068885, + "r2_score": 0.9403530496789464, + "l2_loss": 1.3988661558881252 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..84cf792ebb46543b3d85862aec25c6f2a059c7a3 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21f1c0f5a67a6b13e764185ebed981ec44d5091f39caefac14e168d7e68f56a1 +size 25806 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2aca701ec60deff59e9c329f3b02476a1904753f --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:812644f62f59bb8b3fa3e41ba687588bf4789d79b567f6bdf808d705236ee118 +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d520db2a83ce11fbaa60c9edadf90621d2942e1 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1fadd8e571a2d09c4da2e9c5214e4fd0c1b50d75baa5965c7e057f218523459 +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/config.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..299f0e255d08395b093c773a1cf7e7d84cb69433 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/model.safetensors b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..367c5f21f00fd2a9c9c3597a5d9b95a23bdb7b11 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b58c98b8b9284efafb887782ee376e272860136594caa8a72303d5afdcac7694 +size 37776772 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b40220ec942689e19862d846892782312126b2eb --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e28d1551f2e4051666005721066f130edf022d817fa1fc39f1e6c5d2acca9a42 +size 50496 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..20c19e8ef2162a9ca5962e9bbed8bdab8c62b72b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d94ee5fa7190a71dc93a1b09a3383645bebeab5fac8f7a26a55243e62bc233b9 +size 25895 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..a4d957f6257eefd56875f5c4b080b96c2c7a16d9 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.90002719726563, + "l0_per_token_0.001": 127.90002719726563, + "l0_per_token_0.01": 127.90002719726563, + "l0_per_token_0.05": 127.90002719726563, + "l0_per_token_0.1": 127.90002719726563, + "l0_per_token_0.5": 127.90002719726563, + "l0_per_token_1.0": 127.90002719726563, + "l1_per_token": 775.5630566501216, + "r2_score": 0.9403705726506784, + "l2_loss": 1.3984551995529824 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e826dc099e5de51fa32934528c7bbb0503682b38 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fabf9da94d41bae386ab4766db9548c8eb7d9c4928e2d21c68b47c5658d6bf2 +size 25806 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d29bb1e53bedc7c6bbf164860ddf09688f16a988 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e36aee5441548a428d87b574737662b5fe3142e7966b5894ce263a4bd49bf359 +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6ce8ded7ff1b0a6e4056a06cc51c7495a723fdd --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b9d1e56145e6076b732313c24c9bbb651135f702d8b887f8fc628633ca8ea7d +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/code_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..fd565d75bc3ab7b21a11ecaf71c6c19875118f4c --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 31.99074, + "recon_r2": 0.8981794381114538, + "l1_per_token": 365.0512193890007 + }, + "seed1": { + "l0": 31.908313333333332, + "recon_r2": 0.898061346135489, + "l1_per_token": 364.62938691677573 + }, + "seed2": { + "l0": 31.899991666666665, + "recon_r2": 0.8980410504560813, + "l1_per_token": 363.73484323337397 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6788151185018808, + "mcc_abs": 0.6788479875835933, + "act_iou": 0.3555797449247946, + "act_iou_n_pairs": 6144, + "input_r2": 0.951043418314246, + "matched_mcc_signed": { + "0.6": 0.8135178691093606, + "0.7": 0.8432053357127497, + "0.8": 0.872834989048224, + "0.9": 0.9095104379583612 + }, + "matched_mcc_abs": { + "0.6": 0.8117205497884583, + "0.7": 0.8418240963673066, + "0.8": 0.8723393377987771, + "0.9": 0.9089470863383855 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6758893589112391, + "mcc_abs": 0.6759204669030717, + "act_iou": 0.3538480320289729, + "act_iou_n_pairs": 6144, + "input_r2": 0.9520740988615541, + "matched_mcc_signed": { + "0.6": 0.8139831339687438, + "0.7": 0.8410395571214196, + "0.8": 0.8733304663443476, + "0.9": 0.9134162731447523 + }, + "matched_mcc_abs": { + "0.6": 0.8113403779973313, + "0.7": 0.839350545735074, + "0.8": 0.8728342285796395, + "0.9": 0.9126133439176948 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6739110669574275, + "mcc_abs": 0.6739413021153272, + "act_iou": 0.35135426994710545, + "act_iou_n_pairs": 6144, + "input_r2": 0.9498717600709362, + "matched_mcc_signed": { + "0.6": 0.8133474273000195, + "0.7": 0.8417850895549129, + "0.8": 0.8743065047542785, + "0.9": 0.9143964448760735 + }, + "matched_mcc_abs": { + "0.6": 0.8112580429526928, + "0.7": 0.8411735192291072, + "0.8": 0.8741815692577156, + "0.9": 0.914267186123019 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 31.933014999999997, + "l0_std": 0.05016417300003854, + "recon_r2_mean": 0.898093944901008, + "recon_r2_std": 7.473148884440038e-05, + "l1_per_token_mean": 364.4718165130501, + "l1_per_token_std": 0.6721851455316113, + "input_r2_mean": 0.9509964257489121, + "input_r2_std": 0.001101921169168923, + "mcc_signed_mean": 0.6762051814568492, + "mcc_signed_std": 0.0024672329233172024, + "mcc_abs_mean": 0.6762365855339975, + "mcc_abs_std": 0.002468570196031949, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8136161434593747, + "0.7": 0.8420099941296941, + "0.8": 0.8734906533822834, + "0.9": 0.9124410519930622 + }, + "matched_mcc_signed_band": 0.8603894607411036, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8114396569128276, + "0.7": 0.8407827204438293, + "0.8": 0.8731183785453774, + "0.9": 0.9119425387930331 + }, + "matched_mcc_abs_band": 0.8593208236737668, + "act_iou_mean": 0.35359401563362436, + "act_iou_std": 0.0021241593498602497, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/dict_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..8dfdf5f760405f1982452cc6d0bfc92fb831f1fa --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 32, + "l0_raw": 32.02121372070312, + "recon_r2": 0.8980366766639799, + "l1_per_token": 364.9041291449858, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 20, + "dense_rate": 0.0032552083333333335, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.2123296856880188, + "n_dup_abs": { + "0.5": 234, + "0.6": 133, + "0.7": 102, + "0.8": 74, + "0.9": 38 + }, + "n_dup_clusters_abs": { + "0.5": 116, + "0.6": 66, + "0.7": 51, + "0.8": 37, + "0.9": 19 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2747159004211426 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 32, + "l0_raw": 31.935845947265626, + "recon_r2": 0.8979137417355628, + "l1_per_token": 364.5335010566602, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 18, + "dense_rate": 0.0029296875, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 12, + "0.6": 4, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 6, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.21295468509197235, + "n_dup_abs": { + "0.5": 246, + "0.6": 142, + "0.7": 96, + "0.8": 66, + "0.9": 34 + }, + "n_dup_clusters_abs": { + "0.5": 122, + "0.6": 70, + "0.7": 48, + "0.8": 33, + "0.9": 17 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2743373215198517 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 32, + "l0_raw": 31.92126279296875, + "recon_r2": 0.8979196433601951, + "l1_per_token": 363.6036822997426, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 17, + "dense_rate": 0.0027669270833333335, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.21285521984100342, + "n_dup_abs": { + "0.5": 218, + "0.6": 126, + "0.7": 90, + "0.8": 52, + "0.9": 28 + }, + "n_dup_clusters_abs": { + "0.5": 109, + "0.6": 63, + "0.7": 45, + "0.8": 26, + "0.9": 14 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2729339599609375 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.6951887011528015, + "dcs_abs": 0.6988618969917297, + "frac_sign_flipped": 0.053059895833333336, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.695227324962616, + "0.6": 0.6951733827590942, + "0.7": 0.6951887011528015, + "0.8": 0.6951887011528015, + "0.9": 0.6951887011528015 + }, + "unique_frac_signed": { + "0.5": 0.9993489583333334, + "0.6": 0.999755859375, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6947747030062601, + "0.6": 0.6950036626949441, + "0.7": 0.6951887011528015, + "0.8": 0.6951887011528015, + "0.9": 0.6951887011528015 + }, + "matched_frac_signed": { + "0.5": 0.8126627604166666, + "0.6": 0.7379557291666666, + "0.7": 0.61669921875, + "0.8": 0.4305013020833333, + "0.9": 0.16845703125 + }, + "matched_cos_abs": { + "0.5": 0.6994723677635193, + "0.6": 0.699765145778656, + "0.7": 0.6995293498039246, + "0.8": 0.6993856430053711, + "0.9": 0.699089765548706 + }, + "unique_frac_abs": { + "0.5": 0.9803059895833334, + "0.6": 0.9886881510416666, + "0.7": 0.991943359375, + "0.8": 0.9943033854166666, + "0.9": 0.9970703125 + }, + "dedup_dcs_abs": { + "0.5": 0.685696951666614, + "0.6": 0.6918495081433017, + "0.7": 0.6938934932259144, + "0.8": 0.6954015125520527, + "0.9": 0.6970416510012001 + }, + "matched_frac_abs": { + "0.5": 0.8009440104166666, + "0.6": 0.7337239583333334, + "0.7": 0.6160481770833334, + "0.8": 0.4296875, + "0.9": 0.16796875 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.6924082636833191, + "dcs_abs": 0.6958296298980713, + "frac_sign_flipped": 0.050618489583333336, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6923553347587585, + "0.6": 0.6923787593841553, + "0.7": 0.6924082636833191, + "0.8": 0.6924082636833191, + "0.9": 0.6924082636833191 + }, + "unique_frac_signed": { + "0.5": 0.9996744791666666, + "0.6": 0.9999186197916666, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.692129958673225, + "0.6": 0.6923224134564709, + "0.7": 0.6924082636833191, + "0.8": 0.6924082636833191, + "0.9": 0.6924082636833191 + }, + "matched_frac_signed": { + "0.5": 0.8064778645833334, + "0.6": 0.7333984375, + "0.7": 0.6124674479166666, + "0.8": 0.4207356770833333, + "0.9": 0.16813151041666666 + }, + "matched_cos_abs": { + "0.5": 0.6967424750328064, + "0.6": 0.6966360807418823, + "0.7": 0.6966063976287842, + "0.8": 0.6964092254638672, + "0.9": 0.695866048336029 + }, + "unique_frac_abs": { + "0.5": 0.9815266927083334, + "0.6": 0.9894205729166666, + "0.7": 0.9921875, + "0.8": 0.994873046875, + "0.9": 0.997314453125 + }, + "dedup_dcs_abs": { + "0.5": 0.683871337188369, + "0.6": 0.6892660701220544, + "0.7": 0.6911641601473093, + "0.8": 0.6928387680090964, + "0.9": 0.6939972674445016 + }, + "matched_frac_abs": { + "0.5": 0.7962239583333334, + "0.6": 0.73046875, + "0.7": 0.6116536458333334, + "0.8": 0.4191080729166667, + "0.9": 0.16715494791666666 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.6885316371917725, + "dcs_abs": 0.6921456456184387, + "frac_sign_flipped": 0.050130208333333336, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6886177062988281, + "0.6": 0.688620924949646, + "0.7": 0.6885316371917725, + "0.8": 0.6885316371917725, + "0.9": 0.6885316371917725 + }, + "unique_frac_signed": { + "0.5": 0.9993489583333334, + "0.6": 0.9998372395833334, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6881693874796232, + "0.6": 0.6885088447209758, + "0.7": 0.6885316371917725, + "0.8": 0.6885316371917725, + "0.9": 0.6885316371917725 + }, + "matched_frac_signed": { + "0.5": 0.8059895833333334, + "0.6": 0.73193359375, + "0.7": 0.60986328125, + "0.8": 0.4088541666666667, + "0.9": 0.14697265625 + }, + "matched_cos_abs": { + "0.5": 0.6929879188537598, + "0.6": 0.6930370926856995, + "0.7": 0.6926603317260742, + "0.8": 0.6926648020744324, + "0.9": 0.6922714114189148 + }, + "unique_frac_abs": { + "0.5": 0.9810384114583334, + "0.6": 0.989013671875, + "0.7": 0.992431640625, + "0.8": 0.9951985677083334, + "0.9": 0.9974772135416666 + }, + "dedup_dcs_abs": { + "0.5": 0.6798477670721089, + "0.6": 0.6854231597826583, + "0.7": 0.6874180294107646, + "0.8": 0.6893390189264513, + "0.9": 0.6905249584766958 + }, + "matched_frac_abs": { + "0.5": 0.7939453125, + "0.6": 0.7283528645833334, + "0.7": 0.607421875, + "0.8": 0.4072265625, + "0.9": 0.146484375 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 18, + "n_dense_std": 1.5275252316519465, + "l0_mean": 32, + "l0_std": 0.05399153187597117, + "recon_r2_mean": 0.897956687253246, + "recon_r2_std": null, + "l1_per_token_mean": 364.34710416712954, + "l1_per_token_std": 0.6699613762357992, + "babel_mu1_signed_mean": 0.21271319687366486, + "babel_mu1_signed_std": 0.00033583322672765955, + "n_dup_signed_per_threshold": { + "0.5": 6.666666666666667, + "0.6": 2.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.5, + "n_dup_signed_band_std": 0.5, + "n_dup_clusters_signed_per_threshold": { + "0.5": 3.3333333333333335, + "0.6": 1.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.25, + "n_dup_clusters_signed_band_std": 0.25, + "largest_dup_signed_per_threshold": { + "0.5": 2.0, + "0.6": 1.6666666666666667, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.1666666666666667, + "largest_dup_signed_band_std": 0.14433756729740646, + "dcs_signed_mean": 0.692042867342631, + "dcs_signed_std": 0.003343540191122148, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.6916913497197027, + "0.6": 0.6919449736241302, + "0.7": 0.692042867342631, + "0.8": 0.692042867342631, + "0.9": 0.692042867342631 + }, + "dedup_dcs_signed_band": 0.6920183939130058, + "dedup_dcs_signed_band_std": 0.0033236003251259512, + "matched_cos_signed_per_threshold": { + "0.5": 0.6920667886734009, + "0.6": 0.6920576890309652, + "0.7": 0.692042867342631, + "0.8": 0.692042867342631, + "0.9": 0.692042867342631 + }, + "matched_cos_signed_band": 0.6920465727647145, + "matched_cos_signed_band_std": 0.0033296252262420894, + "unique_frac_signed_per_threshold": { + "0.5": 0.9994574652777778, + "0.6": 0.9998372395833334, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9999593098958334, + "unique_frac_signed_band_std": 2.034505208331483e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.8083767361111112, + "0.6": 0.7344292534722222, + "0.7": 0.6130099826388888, + "0.8": 0.4200303819444444, + "0.9": 0.1611870659722222 + }, + "matched_frac_signed_band": 0.4821641710069444, + "matched_frac_signed_band_std": 0.0071212719935890845, + "babel_mu1_abs_mean": 0.2739957273006439, + "babel_mu1_abs_std": 0.0009387986519355962, + "n_dup_abs_per_threshold": { + "0.5": 232.66666666666666, + "0.6": 133.66666666666666, + "0.7": 96.0, + "0.8": 64.0, + "0.9": 33.333333333333336 + }, + "n_dup_abs_band": 81.74999999999999, + "n_dup_abs_band_std": 6.8053287944081, + "n_dup_clusters_abs_per_threshold": { + "0.5": 115.66666666666667, + "0.6": 66.33333333333333, + "0.7": 48.0, + "0.8": 32.0, + "0.9": 16.666666666666668 + }, + "n_dup_clusters_abs_band": 40.74999999999999, + "n_dup_clusters_abs_band_std": 3.307189138830738, + "largest_dup_abs_per_threshold": { + "0.5": 2.6666666666666665, + "0.6": 2.6666666666666665, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.1666666666666665, + "largest_dup_abs_band_std": 0.14433756729740643, + "dcs_abs_mean": 0.69561239083608, + "dcs_abs_std": 0.0033633915524139517, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6831386853090308, + "0.6": 0.6888462460160047, + "0.7": 0.6908252275946628, + "0.8": 0.6925264331625335, + "0.9": 0.6938546256407991 + }, + "dedup_dcs_abs_band": 0.6915131331035, + "dedup_dcs_abs_band_std": 0.0031959464592871004, + "matched_cos_abs_per_threshold": { + "0.5": 0.6964009205500284, + "0.6": 0.6964794397354126, + "0.7": 0.6962653597195944, + "0.8": 0.6961532235145569, + "0.9": 0.6957424084345499 + }, + "matched_cos_abs_band": 0.6961601078510284, + "matched_cos_abs_band_std": 0.0033973473571545856, + "unique_frac_abs_per_threshold": { + "0.5": 0.98095703125, + "0.6": 0.989040798611111, + "0.7": 0.9921875, + "0.8": 0.9947916666666666, + "0.9": 0.9972873263888888 + }, + "unique_frac_abs_band": 0.9933268229166666, + "unique_frac_abs_band_std": 0.000284830729166701, + "matched_frac_abs_per_threshold": { + "0.5": 0.7970377604166666, + "0.6": 0.7308485243055557, + "0.7": 0.6117078993055556, + "0.8": 0.4186740451388889, + "0.9": 0.16053602430555555 + }, + "matched_frac_abs_band": 0.4804416232638889, + "matched_frac_abs_band_std": 0.007383245283146693, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/config.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4799215e562b7b652f2a6388902ec651236cf521 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/model.safetensors b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c51abb580777ff7cb88565cc2b522c4d309d0281 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddf38219a774971b44fbdc91f28c15d301b9c07eb8b2025e9695bb7ae6738bf7 +size 37776772 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..77218a12bef86a17b2a5b2ad07e86edd6bd068f0 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84a09b81d93c7b0170f349b8ae9e45596737445149764479e5fb33361a4f581e +size 50496 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..122ca2aa58a4acb41520da2fdbc3dfac98a47f86 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b606c33b006df3615fce1adaa6723a4845e285742bbe50d566f3be3187fae56b +size 25895 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..d63ca544c021a73554039d5ad11fb57677975f5d --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.02121372070312, + "l0_per_token_0.001": 32.02121372070312, + "l0_per_token_0.01": 32.02121372070312, + "l0_per_token_0.05": 32.02121372070312, + "l0_per_token_0.1": 32.02121372070312, + "l0_per_token_0.5": 32.02121372070312, + "l0_per_token_1.0": 32.02121372070312, + "l1_per_token": 364.9041291449858, + "r2_score": 0.8980366766639799, + "l2_loss": 2.391288094175893 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4940ea9f109b867e3e5bd0618ff9e0c07048397 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c41d8990dc4bec31b907f92a3d951288856eff1595e669e38600112f0c98f0f7 +size 25806 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5f4abf2425091a4c5c376b2f0a43d14664f2db1 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ecf493578786e4e169cf59444a64393a27d67378826f1cae9dad381f8ea645b +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..25bedbf7fa02cba586064e97444aba8db697d16e --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34b63fa4e66c34edb6acc276dd8cd7f1737045a72b6029cda790b5607e90109c +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/config.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..77133a8368ff520168fe29f6cc0b96c43ebb64da --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/model.safetensors b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e1a21b7faa598f592f20472100a1aaa35b69ecd5 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7d8ac1b89b47f71aa20d6637c2d7265ea9d08820461020f98f10f41d0ecbd42 +size 37776772 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ecf33420c981447f72fcc8dd927cdcf3d3000aeb --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d210c12aa27ff1921128fbebe3cbfa2993b67c8175ddc49b2c62395cfcb857aa +size 50496 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a4d5ae9b1217d817c1bf3504ceb3784bd9796c7 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f390cf2fe57c4e39ebfb04be4adf3baa123e5b1ec3491e004643130b247a485e +size 25895 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..9868b31f232fd8369750f6c570ee967119ff4a9b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.935845947265626, + "l0_per_token_0.001": 31.935845947265626, + "l0_per_token_0.01": 31.935845947265626, + "l0_per_token_0.05": 31.935845947265626, + "l0_per_token_0.1": 31.935845947265626, + "l0_per_token_0.5": 31.935845947265626, + "l0_per_token_1.0": 31.935845947265626, + "l1_per_token": 364.5335010566602, + "r2_score": 0.8979137417355628, + "l2_loss": 2.3941712174506544 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c21dc37488d49b27d114fb3bd19188525c7b7e44 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a439542afb236b099437987ae3d58d13d742b4013412430d6028c387a104e5c0 +size 25806 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b523f54480cb77289db1017cf550e751c95aaf32 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84803d4ac8232c695674741e6788a11a46253a0b8ba4433498636c7ed2471049 +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad0d8938220fff0e4ca1896b4eee60b58d7b7694 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65fcc1a396d2570ede7e559e40ff3d09609d811a47228ebee68b294acd507ab9 +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/config.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..52e803cf6d9f59ccf0bec19d110903bdbe209e58 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/model.safetensors b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7cf8183b6063fdf7f9709858c4f59486672e590c --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:395511bb6df55596dc6aed5c7e5913b8fb72712045a5ebe23fd0903368e527c9 +size 37776772 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..143f837655235faae70f3239c645526dfa1227c5 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af6ae28b22ad1f65969ffe37a69a368cf04fe45be42b4fc34f32a56dca5699c1 +size 50496 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..421dffa2733cd7ed388caedcd0d841e7735feaee --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3adb5fbd2d81cca4d54191d8f09b65c6d30fe525eacc280de793ca5cc78cf0b8 +size 25895 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..aca155bb4cf3c70a08dab315da00eb0ce59e71ed --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.92126279296875, + "l0_per_token_0.001": 31.92126279296875, + "l0_per_token_0.01": 31.92126279296875, + "l0_per_token_0.05": 31.92126279296875, + "l0_per_token_0.1": 31.92126279296875, + "l0_per_token_0.5": 31.92126279296875, + "l0_per_token_1.0": 31.92126279296875, + "l1_per_token": 363.6036822997426, + "r2_score": 0.8979196433601951, + "l2_loss": 2.3940328099894432 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ed0c1f05419dae98536624c8aad13efba393256 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d240451cb59ef2e99a3c65dee5686f7d09470ad98a79dc582f1787c784df0498 +size 25806 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..48602960e0ec2d5647432a70c16e42b424f9628d --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1aa05da66d4c2d2d281338cb25828eb4af7cd06f1f9a8d5042e033bd05ea6861 +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c4a51254ab2be240ae92a225ae292cd2adc0204 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c140897378ee0c262370f589d42e26f569dc7480c43c027d47346a9ff3b14eef +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/code_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..8e3d7cc8104bfdf1a584d52a751df80dcc467ee4 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 64.02777166666667, + "recon_r2": 0.9189086803990298, + "l1_per_token": 515.098729703722 + }, + "seed1": { + "l0": 63.812976666666664, + "recon_r2": 0.9188050402439039, + "l1_per_token": 515.4337396047922 + }, + "seed2": { + "l0": 63.92830333333333, + "recon_r2": 0.9189538305972654, + "l1_per_token": 514.5160000626851 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6719911515910386, + "mcc_abs": 0.6720299292910127, + "act_iou": 0.31871657685519145, + "act_iou_n_pairs": 6144, + "input_r2": 0.9507322932746294, + "matched_mcc_signed": { + "0.6": 0.8164355558684477, + "0.7": 0.8484356832555391, + "0.8": 0.8801018239898769, + "0.9": 0.9298187237238206 + }, + "matched_mcc_abs": { + "0.6": 0.8141394146981019, + "0.7": 0.8478172651936222, + "0.8": 0.8804675295784662, + "0.9": 0.9302960197675872 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6726704259176802, + "mcc_abs": 0.6727016339844134, + "act_iou": 0.31763756010703786, + "act_iou_n_pairs": 6144, + "input_r2": 0.9495939407378191, + "matched_mcc_signed": { + "0.6": 0.815708346515548, + "0.7": 0.8480430555381947, + "0.8": 0.8825273299518422, + "0.9": 0.9317155443741855 + }, + "matched_mcc_abs": { + "0.6": 0.814900010277806, + "0.7": 0.8482465036711808, + "0.8": 0.8818337203530999, + "0.9": 0.9327070353759569 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6729400294325188, + "mcc_abs": 0.6729794197131239, + "act_iou": 0.3177377166347554, + "act_iou_n_pairs": 6144, + "input_r2": 0.949546892247682, + "matched_mcc_signed": { + "0.6": 0.8167937678315377, + "0.7": 0.8457106733728572, + "0.8": 0.880125332643883, + "0.9": 0.9274353106678102 + }, + "matched_mcc_abs": { + "0.6": 0.8149525953900999, + "0.7": 0.8451538332214764, + "0.8": 0.8806978342257755, + "0.9": 0.9272878660092401 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 63.92301722222222, + "l0_std": 0.10749502423023824, + "recon_r2_mean": 0.918889183746733, + "recon_r2_std": 7.628716734452823e-05, + "l1_per_token_mean": 515.0161564570664, + "l1_per_token_std": 0.46440846523743623, + "input_r2_mean": 0.9499577087533768, + "input_r2_std": 0.000671222225145364, + "mcc_signed_mean": 0.6725338689804126, + "mcc_signed_std": 0.0004889561711953229, + "mcc_abs_mean": 0.6725703276628501, + "mcc_abs_std": 0.0004881741779141184, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8163125567385111, + "0.7": 0.847396470722197, + "0.8": 0.8809181621952007, + "0.9": 0.9296565262552722 + }, + "matched_mcc_signed_band": 0.8685709289777953, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8146640067886692, + "0.7": 0.8470725340287598, + "0.8": 0.8809996947191139, + "0.9": 0.9300969737175947 + }, + "matched_mcc_abs_band": 0.8682083023135344, + "act_iou_mean": 0.31803061786566156, + "act_iou_std": 0.0005961649385855729, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/dict_metrics_step_final.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..291c7130bd1dc1b20cc87b79c6eb0c0afd903253 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 64.05995439453125, + "recon_r2": 0.9187922981886927, + "l1_per_token": 514.9229289249525, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 107, + "dense_rate": 0.017415364583333332, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.17871303856372833, + "n_dup_abs": { + "0.5": 326, + "0.6": 266, + "0.7": 224, + "0.8": 174, + "0.9": 86 + }, + "n_dup_clusters_abs": { + "0.5": 163, + "0.6": 133, + "0.7": 112, + "0.8": 87, + "0.9": 43 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2624035179615021 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 63.8338287109375, + "recon_r2": 0.9186987362708899, + "l1_per_token": 515.1930015483604, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 114, + "dense_rate": 0.0185546875, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.17848019301891327, + "n_dup_abs": { + "0.5": 334, + "0.6": 238, + "0.7": 204, + "0.8": 168, + "0.9": 68 + }, + "n_dup_clusters_abs": { + "0.5": 167, + "0.6": 119, + "0.7": 102, + "0.8": 84, + "0.9": 34 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2627171277999878 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 63.96698076171875, + "recon_r2": 0.9188538777526928, + "l1_per_token": 514.3508933209006, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 107, + "dense_rate": 0.017415364583333332, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.17740975320339203, + "n_dup_abs": { + "0.5": 329, + "0.6": 255, + "0.7": 223, + "0.8": 184, + "0.9": 82 + }, + "n_dup_clusters_abs": { + "0.5": 164, + "0.6": 127, + "0.7": 111, + "0.8": 92, + "0.9": 41 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2622268497943878 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.6577297449111938, + "dcs_abs": 0.6615012884140015, + "frac_sign_flipped": 0.05078125, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6578187942504883, + "0.6": 0.6577297449111938, + "0.7": 0.6577297449111938, + "0.8": 0.6577297449111938, + "0.9": 0.6577297449111938 + }, + "unique_frac_signed": { + "0.5": 0.9999186197916666, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6577652608199666, + "0.6": 0.6577297449111938, + "0.7": 0.6577297449111938, + "0.8": 0.6577297449111938, + "0.9": 0.6577297449111938 + }, + "matched_frac_signed": { + "0.5": 0.7869466145833334, + "0.6": 0.6949869791666666, + "0.7": 0.5465494791666666, + "0.8": 0.3087565104166667, + "0.9": 0.08805338541666667 + }, + "matched_cos_abs": { + "0.5": 0.663815438747406, + "0.6": 0.6644656658172607, + "0.7": 0.6640549898147583, + "0.8": 0.6631742715835571, + "0.9": 0.6624273657798767 + }, + "unique_frac_abs": { + "0.5": 0.97314453125, + "0.6": 0.9794921875, + "0.7": 0.9825846354166666, + "0.8": 0.986083984375, + "0.9": 0.9937337239583334 + }, + "dedup_dcs_abs": { + "0.5": 0.6459883639763575, + "0.6": 0.6508389285299927, + "0.7": 0.6524902300637525, + "0.8": 0.6539455280581024, + "0.9": 0.6582764130483459 + }, + "matched_frac_abs": { + "0.5": 0.7747395833333334, + "0.6": 0.68896484375, + "0.7": 0.54248046875, + "0.8": 0.30517578125, + "0.9": 0.08658854166666667 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.6554637551307678, + "dcs_abs": 0.6584895253181458, + "frac_sign_flipped": 0.053059895833333336, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6554926633834839, + "0.6": 0.6554926633834839, + "0.7": 0.6554926633834839, + "0.8": 0.655463695526123, + "0.9": 0.655463695526123 + }, + "unique_frac_signed": { + "0.5": 0.9999186197916666, + "0.6": 0.9999186197916666, + "0.7": 0.9999186197916666, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6554393192539767, + "0.6": 0.6554393192539767, + "0.7": 0.6554393192539767, + "0.8": 0.655463695526123, + "0.9": 0.655463695526123 + }, + "matched_frac_signed": { + "0.5": 0.7849934895833334, + "0.6": 0.6896158854166666, + "0.7": 0.53369140625, + "0.8": 0.2996419270833333, + "0.9": 0.08154296875 + }, + "matched_cos_abs": { + "0.5": 0.6612498760223389, + "0.6": 0.6612845063209534, + "0.7": 0.6609324216842651, + "0.8": 0.6604604125022888, + "0.9": 0.6594077348709106 + }, + "unique_frac_abs": { + "0.5": 0.9733072916666666, + "0.6": 0.978759765625, + "0.7": 0.9817708333333334, + "0.8": 0.9854329427083334, + "0.9": 0.9931640625 + }, + "dedup_dcs_abs": { + "0.5": 0.6435993259462217, + "0.6": 0.6472386684181402, + "0.7": 0.648884174413979, + "0.8": 0.6508394478344902, + "0.9": 0.6549000648083165 + }, + "matched_frac_abs": { + "0.5": 0.7734375, + "0.6": 0.68408203125, + "0.7": 0.52978515625, + "0.8": 0.29736328125, + "0.9": 0.08040364583333333 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.6563121676445007, + "dcs_abs": 0.6593895554542542, + "frac_sign_flipped": 0.048990885416666664, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6563819050788879, + "0.6": 0.656387448310852, + "0.7": 0.656387448310852, + "0.8": 0.6563122868537903, + "0.9": 0.6563122868537903 + }, + "unique_frac_signed": { + "0.5": 0.9998372395833334, + "0.6": 0.9999186197916666, + "0.7": 0.9999186197916666, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6562750720865249, + "0.6": 0.656334031363561, + "0.7": 0.656334031363561, + "0.8": 0.6563122868537903, + "0.9": 0.6563122868537903 + }, + "matched_frac_signed": { + "0.5": 0.7872721354166666, + "0.6": 0.6935221354166666, + "0.7": 0.5390625, + "0.8": 0.2960611979166667, + "0.9": 0.08203125 + }, + "matched_cos_abs": { + "0.5": 0.6616686582565308, + "0.6": 0.662419855594635, + "0.7": 0.6623929738998413, + "0.8": 0.6618390083312988, + "0.9": 0.660511314868927 + }, + "unique_frac_abs": { + "0.5": 0.9729817708333334, + "0.6": 0.9798990885416666, + "0.7": 0.9825846354166666, + "0.8": 0.9856770833333334, + "0.9": 0.993896484375 + }, + "dedup_dcs_abs": { + "0.5": 0.643791542815355, + "0.6": 0.6491046127290853, + "0.7": 0.6508571587619372, + "0.8": 0.6523595433682203, + "0.9": 0.6564798737381352 + }, + "matched_frac_abs": { + "0.5": 0.77490234375, + "0.6": 0.6881510416666666, + "0.7": 0.5345052083333334, + "0.8": 0.29248046875, + "0.9": 0.08138020833333333 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 109, + "n_dense_std": 4.041451884327381, + "l0_mean": 64, + "l0_std": 0.11365619927830771, + "recon_r2_mean": 0.9187816374040918, + "recon_r2_std": null, + "l1_per_token_mean": 514.8222745980711, + "l1_per_token_std": 0.4299825594629328, + "babel_mu1_signed_mean": 0.17820099492867789, + "babel_mu1_signed_std": 0.0006950552947859976, + "n_dup_signed_per_threshold": { + "0.5": 1.3333333333333333, + "0.6": 0.6666666666666666, + "0.7": 0.6666666666666666, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.3333333333333333, + "n_dup_signed_band_std": 0.5773502691896258, + "n_dup_clusters_signed_per_threshold": { + "0.5": 0.6666666666666666, + "0.6": 0.3333333333333333, + "0.7": 0.3333333333333333, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.16666666666666666, + "n_dup_clusters_signed_band_std": 0.2886751345948129, + "largest_dup_signed_per_threshold": { + "0.5": 1.6666666666666667, + "0.6": 1.3333333333333333, + "0.7": 1.3333333333333333, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.1666666666666665, + "largest_dup_signed_band_std": 0.2886751345948129, + "dcs_signed_mean": 0.6565018892288208, + "dcs_signed_std": 0.0011448463350670623, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.6564932173868228, + "0.6": 0.6565010318429105, + "0.7": 0.6565010318429105, + "0.8": 0.6565019090970358, + "0.9": 0.6565019090970358 + }, + "dedup_dcs_signed_band": 0.6565014704699732, + "dedup_dcs_signed_band_std": 0.0011495380623810936, + "matched_cos_signed_per_threshold": { + "0.5": 0.65656445423762, + "0.6": 0.6565366188685099, + "0.7": 0.6565366188685099, + "0.8": 0.6565019090970358, + "0.9": 0.6565019090970358 + }, + "matched_cos_signed_band": 0.6565192639827728, + "matched_cos_signed_band_std": 0.0011353008880764122, + "unique_frac_signed_per_threshold": { + "0.5": 0.9998914930555555, + "0.6": 0.9999457465277777, + "0.7": 0.9999457465277777, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9999728732638888, + "unique_frac_signed_band_std": 2.3492442594021645e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.7864040798611112, + "0.6": 0.6927083333333334, + "0.7": 0.5397677951388888, + "0.8": 0.3014865451388889, + "0.9": 0.08387586805555557 + }, + "matched_frac_signed_band": 0.4044596354166667, + "matched_frac_signed_band_std": 0.004506876789062661, + "babel_mu1_abs_mean": 0.26244916518529254, + "babel_mu1_abs_std": 0.00024830602585037377, + "n_dup_abs_per_threshold": { + "0.5": 329.6666666666667, + "0.6": 253.0, + "0.7": 217.0, + "0.8": 175.33333333333334, + "0.9": 78.66666666666667 + }, + "n_dup_abs_band": 181.0, + "n_dup_abs_band_std": 9.987492177719089, + "n_dup_clusters_abs_per_threshold": { + "0.5": 164.66666666666666, + "0.6": 126.33333333333333, + "0.7": 108.33333333333333, + "0.8": 87.66666666666667, + "0.9": 39.333333333333336 + }, + "n_dup_clusters_abs_band": 90.41666666666666, + "n_dup_clusters_abs_band_std": 4.932882862316247, + "largest_dup_abs_per_threshold": { + "0.5": 2.3333333333333335, + "0.6": 2.3333333333333335, + "0.7": 2.3333333333333335, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.166666666666667, + "largest_dup_abs_band_std": 0.2886751345948129, + "dcs_abs_mean": 0.6597934563954672, + "dcs_abs_std": 0.001545972578710289, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6444597442459781, + "0.6": 0.6490607365590727, + "0.7": 0.6507438544132229, + "0.8": 0.652381506420271, + "0.9": 0.6565521171982659 + }, + "dedup_dcs_abs_band": 0.6521845536477081, + "dedup_dcs_abs_band_std": 0.0017111473473652617, + "matched_cos_abs_per_threshold": { + "0.5": 0.6622446576754252, + "0.6": 0.6627233425776163, + "0.7": 0.6624601284662882, + "0.8": 0.6618245641390482, + "0.9": 0.6607821385065714 + }, + "matched_cos_abs_band": 0.661947543422381, + "matched_cos_abs_band_std": 0.0015107638483132623, + "unique_frac_abs_per_threshold": { + "0.5": 0.97314453125, + "0.6": 0.9793836805555555, + "0.7": 0.9823133680555555, + "0.8": 0.9857313368055557, + "0.9": 0.9935980902777778 + }, + "unique_frac_abs_band": 0.985256618923611, + "unique_frac_abs_band_std": 0.0004116208470456684, + "matched_frac_abs_per_threshold": { + "0.5": 0.7743598090277778, + "0.6": 0.6870659722222222, + "0.7": 0.5355902777777778, + "0.8": 0.29833984375, + "0.9": 0.0827907986111111 + }, + "matched_frac_abs_band": 0.4009467230902778, + "matched_frac_abs_band_std": 0.004249210774728795, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/config.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8fcb63e0a3fa994dbbcc69d24b9250bcc4ed6262 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/model.safetensors b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..88340f103fe63a05ce533efa15a7983ae1bd1f06 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6710f74436ccb8bbe34889df2dfbc601fd1bd85b4f5d77977136566293a22d9 +size 37776772 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a68430000e5bbb03a22f84630b82610ec7fff1eb --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2b52c34f0ae1d3a0591a1e1a99e5c8c8c053a6735d71d471298bbea35f8771f +size 50496 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e85481471380d784097a767f01706118bcfd602 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90a21f858e3ccdc8723ed28a4d31e1fc7055557145362c619c2536d958e9bd81 +size 25895 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..2ea5b72b90aa6e3cde0ad3d3df35a8c8330b5bfa --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.05995439453125, + "l0_per_token_0.001": 64.05995439453125, + "l0_per_token_0.01": 64.05995439453125, + "l0_per_token_0.05": 64.05995439453125, + "l0_per_token_0.1": 64.05995439453125, + "l0_per_token_0.5": 64.05995439453125, + "l0_per_token_1.0": 64.05995439453125, + "l1_per_token": 514.9229289249525, + "r2_score": 0.9187922981886927, + "l2_loss": 1.9045182536548835 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0dfc9b9b8e74f562a01790f2537544fe681cc685 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3f0a3fa9c7bef319e07d0919b0582bec3762532bcbae7098e3999425e16163a +size 25806 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..da7cab40dfcd1e91da0e3bb7b6ed90cd6bf3acb7 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa067fa7d262d6b4ffa1201e55dc184e336fc481b4fe4a92ffbf413adc98d9b1 +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2da85a5e00ced3bd364ea3c55eba506cc75bcbcf --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f51b3e2f76e194c00576ddc0d403bba73e422fa190fd741454b53be1001ae618 +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/config.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b51a95f2c49d8264c95d2df553e8f47969767674 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/model.safetensors b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cf9632c0414d406f15ed0bfdae9d430d5e2d99f5 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e35de6d3dc501643243f599a8c9ef79b48d62633f9c74262fd9e84920cf38e0b +size 37776772 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d38fd184627d2dfbe599d8da87ce2ca4eb8a6018 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31c0e04862a59972f48806c380ddabf0576c8fa30b96ce6511927e1ba04c3714 +size 50496 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..056d62cce6118e754334c5aa8ba74d23f9e392ab --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88642662c2eda9e8c93014f293648daa6b3846fade54954eb5ff0e98f81755ae +size 25895 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..9f23cc5ec35592c5d7ad66d284e41cb5f7334741 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.8338287109375, + "l0_per_token_0.001": 63.8338287109375, + "l0_per_token_0.01": 63.8338287109375, + "l0_per_token_0.05": 63.8338287109375, + "l0_per_token_0.1": 63.8338287109375, + "l0_per_token_0.5": 63.8338287109375, + "l0_per_token_1.0": 63.8338287109375, + "l1_per_token": 515.1930015483604, + "r2_score": 0.9186987362708899, + "l2_loss": 1.9067125083416658 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac39825d5036ba5d4fb97a0a679ba18347b0889b --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85179dc63bc86c90c26b3230fe483ec8e76dcfdd38bd29cfa801d1c389072c2a +size 25806 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d7ff9a5a28c1f1e173052bd2b633552572ace3e --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2054030646c2c818e9c68e56ec1394e1eda9f727b2f51b348a8017dcd8971b8a +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b9b05b32b33c21d1de5c8d333faecbc08437afb --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b8cbb438494c0e5f12be6de6c4754636589a6318685e6ee3586f499b81736fb +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/config.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d79b55bff820762f7c49c954267bbf9d1b1911c4 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/model.safetensors b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..da5cb2d43df6926e951b26f9c18bcb25e46c7953 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e676ef7ef05ae10619aa7ce25347c54d96436250b4d25235861c2031c340af34 +size 37776772 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2aea3e8aa6180033e0c3e9afe8c5771ce7bc2f1d --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a3632e9640e452a3b639543cd00ade01cfead1c469d0bf11d82f8422800dbf3 +size 50496 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..455e9097defc38e9b85a674310d3239332bf62fd --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40e896d16c0c8f89fc4f8821275ec1e24b64a2299628e2233a6c56d8d1d599d4 +size 25895 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..8843fc07b9c41d19154533cf66f6621175aad20e --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.96698076171875, + "l0_per_token_0.001": 63.96698076171875, + "l0_per_token_0.01": 63.96698076171875, + "l0_per_token_0.05": 63.96698076171875, + "l0_per_token_0.1": 63.96698076171875, + "l0_per_token_0.5": 63.96698076171875, + "l0_per_token_1.0": 63.96698076171875, + "l1_per_token": 514.3508933209006, + "r2_score": 0.9188538777526928, + "l2_loss": 1.9030740630045593 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..306242b85689674093cd8f425a6fff69ebb189ff --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa08669bed7899867c9a67d3dd73a5a1f7e2b43d606d693706a636f1b913e23d +size 25806 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8166a324bcc81fafcfc2a57ab9b41218aedb126 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06b6e41a0382720f8d58a7725c9a0064173930a3a312c395ab3a8a7747616dac +size 25811 diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ceff8604b22e949b1345250706ca2c4d49e30d33 --- /dev/null +++ b/lm/gpt2-small_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9e37f9725fc922689c60bf76d92475e457bb324a6e1e93fdaa9ae2190aed236 +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/code_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x16/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..bf52b6c10723807f407a81298560e2eb53195b24 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 261.21647, + "recon_r2": 0.9847891900024385, + "l1_per_token": 802.3347534700799 + }, + "seed1": { + "l0": 259.5063866666667, + "recon_r2": 0.9847295416325721, + "l1_per_token": 801.1025795754 + }, + "seed2": { + "l0": 261.92269666666664, + "recon_r2": 0.9848399666541547, + "l1_per_token": 801.1532248487391 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.4838865486807893, + "mcc_abs": 0.4838936963635519, + "act_iou": 0.07724485732967691, + "act_iou_n_pairs": 12288, + "input_r2": 0.986300313248124, + "matched_mcc_signed": { + "0.6": 0.5669469130603053, + "0.7": 0.5792228421668155, + "0.8": 0.587894549012978, + "0.9": 0.6093967881892141 + }, + "matched_mcc_abs": { + "0.6": 0.5641969852384354, + "0.7": 0.5777336329062613, + "0.8": 0.5873191858114533, + "0.9": 0.6092722746776401 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.48617790817253054, + "mcc_abs": 0.4861868387082062, + "act_iou": 0.07751851479623832, + "act_iou_n_pairs": 12288, + "input_r2": 0.9863939795370449, + "matched_mcc_signed": { + "0.6": 0.56719229332494, + "0.7": 0.5801780364736703, + "0.8": 0.588202794611136, + "0.9": 0.6097011834566746 + }, + "matched_mcc_abs": { + "0.6": 0.5652036450506829, + "0.7": 0.578832735521309, + "0.8": 0.5877368393819278, + "0.9": 0.6101164550144322 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.48413444284111223, + "mcc_abs": 0.48414145967380245, + "act_iou": 0.0766522527661015, + "act_iou_n_pairs": 12288, + "input_r2": 0.9863100790127616, + "matched_mcc_signed": { + "0.6": 0.5657559649728287, + "0.7": 0.5808013478311531, + "0.8": 0.589705540257335, + "0.9": 0.6124598467073248 + }, + "matched_mcc_abs": { + "0.6": 0.5631692931558949, + "0.7": 0.5801305375231778, + "0.8": 0.5899832379324904, + "0.9": 0.6124261146745023 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 260.88185111111113, + "l0_std": 1.2424233797807718, + "recon_r2_mean": 0.9847862327630551, + "recon_r2_std": 5.527187618002564e-05, + "l1_per_token_mean": 801.5301859647398, + "l1_per_token_std": 0.6972358918723081, + "input_r2_mean": 0.9863347905993102, + "input_r2_std": 5.1491167226673785e-05, + "mcc_signed_mean": 0.48473296656481074, + "mcc_signed_std": 0.0012574796483325594, + "mcc_abs_mean": 0.48474066491518686, + "mcc_abs_std": 0.0012585351183771539, + "matched_mcc_signed_per_threshold": { + "0.6": 0.5666317237860247, + "0.7": 0.5800674088238796, + "0.8": 0.5886009612938163, + "0.9": 0.6105192727844045 + }, + "matched_mcc_signed_band": 0.5864548416720313, + "matched_mcc_abs_per_threshold": { + "0.6": 0.5641899744816711, + "0.7": 0.5788989686502494, + "0.8": 0.5883464210419572, + "0.9": 0.6106049481221916 + }, + "matched_mcc_abs_band": 0.5855100780740172, + "act_iou_mean": 0.07713854163067224, + "act_iou_std": 0.00044280892843903615, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/dict_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x16/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..3d2c2e647b27d5182b239ed7a1c3c3fa75e2320a --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 7989, + "0.6": 5686, + "0.7": 3713, + "0.8": 2032, + "0.9": 716 + }, + "n_dup_clusters_signed": { + "0.5": 2725, + "0.6": 1982, + "0.7": 1329, + "0.8": 744, + "0.9": 260 + }, + "largest_dup_signed": { + "0.5": 69, + "0.6": 44, + "0.7": 26, + "0.8": 16, + "0.9": 10 + }, + "babel_mu1_signed": 0.6284380555152893, + "n_dup_abs": { + "0.5": 8023, + "0.6": 5750, + "0.7": 3769, + "0.8": 2052, + "0.9": 720 + }, + "n_dup_clusters_abs": { + "0.5": 2730, + "0.6": 2006, + "0.7": 1355, + "0.8": 754, + "0.9": 261 + }, + "largest_dup_abs": { + "0.5": 47, + "0.6": 36, + "0.7": 26, + "0.8": 16, + "0.9": 10 + }, + "babel_mu1_abs": 0.6307905316352844 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 7991, + "0.6": 5701, + "0.7": 3669, + "0.8": 2005, + "0.9": 736 + }, + "n_dup_clusters_signed": { + "0.5": 2712, + "0.6": 1994, + "0.7": 1296, + "0.8": 723, + "0.9": 256 + }, + "largest_dup_signed": { + "0.5": 50, + "0.6": 31, + "0.7": 29, + "0.8": 26, + "0.9": 10 + }, + "babel_mu1_signed": 0.6283772587776184, + "n_dup_abs": { + "0.5": 8014, + "0.6": 5768, + "0.7": 3727, + "0.8": 2023, + "0.9": 739 + }, + "n_dup_clusters_abs": { + "0.5": 2718, + "0.6": 2024, + "0.7": 1322, + "0.8": 731, + "0.9": 258 + }, + "largest_dup_abs": { + "0.5": 43, + "0.6": 32, + "0.7": 29, + "0.8": 26, + "0.9": 10 + }, + "babel_mu1_abs": 0.6305112242698669 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 7991, + "0.6": 5734, + "0.7": 3681, + "0.8": 2006, + "0.9": 722 + }, + "n_dup_clusters_signed": { + "0.5": 2738, + "0.6": 2027, + "0.7": 1318, + "0.8": 745, + "0.9": 270 + }, + "largest_dup_signed": { + "0.5": 54, + "0.6": 37, + "0.7": 22, + "0.8": 16, + "0.9": 11 + }, + "babel_mu1_signed": 0.6283814311027527, + "n_dup_abs": { + "0.5": 8024, + "0.6": 5789, + "0.7": 3734, + "0.8": 2021, + "0.9": 724 + }, + "n_dup_clusters_abs": { + "0.5": 2748, + "0.6": 2051, + "0.7": 1342, + "0.8": 751, + "0.9": 271 + }, + "largest_dup_abs": { + "0.5": 55, + "0.6": 40, + "0.7": 22, + "0.8": 16, + "0.9": 11 + }, + "babel_mu1_abs": 0.6304722428321838 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8436322212219238, + "dcs_abs": 0.8466837406158447, + "frac_sign_flipped": 0.021077473958333332, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8275504112243652, + "0.6": 0.8313969373703003, + "0.7": 0.8353726863861084, + "0.8": 0.8393656611442566, + "0.9": 0.8434174656867981 + }, + "unique_frac_signed": { + "0.5": 0.5710042317708334, + "0.6": 0.6984456380208334, + "0.7": 0.8064371744791666, + "0.8": 0.8954264322916666, + "0.9": 0.9619140625 + }, + "dedup_dcs_signed": { + "0.5": 0.4725347868128059, + "0.6": 0.5806855643701663, + "0.7": 0.6736755888462842, + "0.8": 0.7515901993465377, + "0.9": 0.8112951208022423 + }, + "matched_frac_signed": { + "0.5": 0.5017903645833334, + "0.6": 0.5933430989583334, + "0.7": 0.641845703125, + "0.8": 0.6387532552083334, + "0.9": 0.56201171875 + }, + "matched_cos_abs": { + "0.5": 0.8312228918075562, + "0.6": 0.8351380825042725, + "0.7": 0.8387950658798218, + "0.8": 0.8426327705383301, + "0.9": 0.8464468717575073 + }, + "unique_frac_abs": { + "0.5": 0.5691324869791666, + "0.6": 0.6953125, + "0.7": 0.8039143880208334, + "0.8": 0.8946126302083334, + "0.9": 0.9617513020833334 + }, + "dedup_dcs_abs": { + "0.5": 0.4730759516484492, + "0.6": 0.580681947991252, + "0.7": 0.6743194220616715, + "0.8": 0.7538299191510305, + "0.9": 0.8140713810571469 + }, + "matched_frac_abs": { + "0.5": 0.502685546875, + "0.6": 0.5953776041666666, + "0.7": 0.6446126302083334, + "0.8": 0.6398111979166666, + "0.9": 0.56201171875 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8473139405250549, + "dcs_abs": 0.8499342799186707, + "frac_sign_flipped": 0.019205729166666668, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8293575048446655, + "0.6": 0.8332625031471252, + "0.7": 0.8369428515434265, + "0.8": 0.8419058322906494, + "0.9": 0.845782995223999 + }, + "unique_frac_signed": { + "0.5": 0.5720621744791666, + "0.6": 0.6984456380208334, + "0.7": 0.8068440755208334, + "0.8": 0.8962809244791666, + "0.9": 0.9630533854166666 + }, + "dedup_dcs_signed": { + "0.5": 0.47444405764205533, + "0.6": 0.5819885606494306, + "0.7": 0.6752823813173261, + "0.8": 0.7545841376898654, + "0.9": 0.8145341768783206 + }, + "matched_frac_signed": { + "0.5": 0.5034993489583334, + "0.6": 0.595458984375, + "0.7": 0.642822265625, + "0.8": 0.6434733072916666, + "0.9": 0.5736490885416666 + }, + "matched_cos_abs": { + "0.5": 0.8338408470153809, + "0.6": 0.8370147943496704, + "0.7": 0.8400601744651794, + "0.8": 0.8446245789527893, + "0.9": 0.8484957814216614 + }, + "unique_frac_abs": { + "0.5": 0.5699462890625, + "0.6": 0.695556640625, + "0.7": 0.804443359375, + "0.8": 0.8955078125, + "0.9": 0.962890625 + }, + "dedup_dcs_abs": { + "0.5": 0.4752444964251481, + "0.6": 0.582191198511282, + "0.7": 0.6757808288239175, + "0.8": 0.7563679090817459, + "0.9": 0.8170086332829669 + }, + "matched_frac_abs": { + "0.5": 0.5048014322916666, + "0.6": 0.5970865885416666, + "0.7": 0.6463216145833334, + "0.8": 0.6448567708333334, + "0.9": 0.5731608072916666 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8424146771430969, + "dcs_abs": 0.8454044461250305, + "frac_sign_flipped": 0.020263671875, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8259482383728027, + "0.6": 0.8288151025772095, + "0.7": 0.8321582078933716, + "0.8": 0.8369964957237244, + "0.9": 0.8409832119941711 + }, + "unique_frac_signed": { + "0.5": 0.5714518229166666, + "0.6": 0.6983235677083334, + "0.7": 0.8072916666666666, + "0.8": 0.8965250651041666, + "0.9": 0.9620768229166666 + }, + "dedup_dcs_signed": { + "0.5": 0.4719896264529476, + "0.6": 0.5787811194022652, + "0.7": 0.6717943865805864, + "0.8": 0.7503883378206713, + "0.9": 0.8090904567216057 + }, + "matched_frac_signed": { + "0.5": 0.4991048177083333, + "0.6": 0.5921223958333334, + "0.7": 0.6378580729166666, + "0.8": 0.6355794270833334, + "0.9": 0.555908203125 + }, + "matched_cos_abs": { + "0.5": 0.8300459980964661, + "0.6": 0.8329750299453735, + "0.7": 0.8358787894248962, + "0.8": 0.8401848077774048, + "0.9": 0.84405916929245 + }, + "unique_frac_abs": { + "0.5": 0.56982421875, + "0.6": 0.695556640625, + "0.7": 0.8048095703125, + "0.8": 0.895751953125, + "0.9": 0.9619954427083334 + }, + "dedup_dcs_abs": { + "0.5": 0.47298031239188276, + "0.6": 0.5793813135533128, + "0.7": 0.6727232493503834, + "0.8": 0.752597182552563, + "0.9": 0.8119810742355185 + }, + "matched_frac_abs": { + "0.5": 0.5013834635416666, + "0.6": 0.5930989583333334, + "0.7": 0.6405436197916666, + "0.8": 0.6357421875, + "0.9": 0.5555826822916666 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.6283989151318868, + "babel_mu1_signed_std": 3.396070205293077e-05, + "n_dup_signed_per_threshold": { + "0.5": 7990.333333333333, + "0.6": 5707.0, + "0.7": 3687.6666666666665, + "0.8": 2014.3333333333333, + "0.9": 724.6666666666666 + }, + "n_dup_signed_band": 3033.4166666666665, + "n_dup_signed_band_std": 4.932882862316248, + "n_dup_clusters_signed_per_threshold": { + "0.5": 2725.0, + "0.6": 2001.0, + "0.7": 1314.3333333333333, + "0.8": 737.3333333333334, + "0.9": 262.0 + }, + "n_dup_clusters_signed_band": 1078.6666666666665, + "n_dup_clusters_signed_band_std": 11.375228935425138, + "largest_dup_signed_per_threshold": { + "0.5": 57.666666666666664, + "0.6": 37.333333333333336, + "0.7": 25.666666666666668, + "0.8": 19.333333333333332, + "0.9": 10.333333333333334 + }, + "largest_dup_signed_band": 23.166666666666664, + "largest_dup_signed_band_std": 1.4433756729740643, + "dcs_signed_mean": 0.8444536129633585, + "dcs_signed_std": 0.0025508251049653602, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.4729894903026029, + "0.6": 0.5804850814739541, + "0.7": 0.6735841189147322, + "0.8": 0.7521875582856915, + "0.9": 0.8116399181340562 + }, + "dedup_dcs_signed_band": 0.7044741692021086, + "dedup_dcs_signed_band_std": 0.0020467164304809188, + "matched_cos_signed_per_threshold": { + "0.5": 0.8276187181472778, + "0.6": 0.831158181031545, + "0.7": 0.8348245819409689, + "0.8": 0.8394226630528768, + "0.9": 0.8433945576349894 + }, + "matched_cos_signed_band": 0.837199995915095, + "matched_cos_signed_band_std": 0.0023732482633511687, + "unique_frac_signed_per_threshold": { + "0.5": 0.5715060763888888, + "0.6": 0.6984049479166666, + "0.7": 0.8068576388888888, + "0.8": 0.8960774739583334, + "0.9": 0.9623480902777777 + }, + "unique_frac_signed_band": 0.8409220377604166, + "unique_frac_signed_band_std": 0.0003212006307748867, + "matched_frac_signed_per_threshold": { + "0.5": 0.50146484375, + "0.6": 0.5936414930555557, + "0.7": 0.6408420138888888, + "0.8": 0.6392686631944445, + "0.9": 0.5638563368055555 + }, + "matched_frac_signed_band": 0.6094021267361112, + "matched_frac_signed_band_std": 0.004257045162116399, + "babel_mu1_abs_mean": 0.6305913329124451, + "babel_mu1_abs_std": 0.00017360871667800333, + "n_dup_abs_per_threshold": { + "0.5": 8020.333333333333, + "0.6": 5769.0, + "0.7": 3743.3333333333335, + "0.8": 2032.0, + "0.9": 727.6666666666666 + }, + "n_dup_abs_band": 3068.0, + "n_dup_abs_band_std": 4.337337893224368, + "n_dup_clusters_abs_per_threshold": { + "0.5": 2732.0, + "0.6": 2027.0, + "0.7": 1339.6666666666667, + "0.8": 745.3333333333334, + "0.9": 263.3333333333333 + }, + "n_dup_clusters_abs_band": 1093.8333333333333, + "n_dup_clusters_abs_band_std": 10.001041612418845, + "largest_dup_abs_per_threshold": { + "0.5": 48.333333333333336, + "0.6": 36.0, + "0.7": 25.666666666666668, + "0.8": 19.333333333333332, + "0.9": 10.333333333333334 + }, + "largest_dup_abs_band": 22.833333333333332, + "largest_dup_abs_band_std": 1.2332207155790618, + "dcs_abs_mean": 0.8473408222198486, + "dcs_abs_std": 0.0023353084860979947, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.47376692015516, + "0.6": 0.5807514866852822, + "0.7": 0.6742745000786575, + "0.8": 0.7542650035951132, + "0.9": 0.8143536961918775 + }, + "dedup_dcs_abs_band": 0.7059111716377326, + "dedup_dcs_abs_band_std": 0.0018402444972756755, + "matched_cos_abs_per_threshold": { + "0.5": 0.831703245639801, + "0.6": 0.8350426355997721, + "0.7": 0.8382446765899658, + "0.8": 0.8424807190895081, + "0.9": 0.8463339408238729 + }, + "matched_cos_abs_band": 0.8405254930257797, + "matched_cos_abs_band_std": 0.002146270011368372, + "unique_frac_abs_per_threshold": { + "0.5": 0.5696343315972222, + "0.6": 0.6954752604166666, + "0.7": 0.8043891059027778, + "0.8": 0.8952907986111112, + "0.9": 0.9622124565972223 + }, + "unique_frac_abs_band": 0.8393419053819444, + "unique_frac_abs_band_std": 0.0003863328434643635, + "matched_frac_abs_per_threshold": { + "0.5": 0.502956814236111, + "0.6": 0.5951877170138888, + "0.7": 0.6438259548611112, + "0.8": 0.64013671875, + "0.9": 0.5635850694444443 + }, + "matched_frac_abs_band": 0.610683865017361, + "matched_frac_abs_band_std": 0.004561664360585359, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/config.json b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..582c8dce741510337aae6bc33e8db9b021f25bc1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..39760dbba21ca823364c989ae4c76afe3d6a32a1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2e1242727f192d221c9f5b036a3c7667c7fd6ff55c6f2bb290d5588852aa57c +size 75550008 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..302a103cec575f786c765105d88256ee3c0cd1ad --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a22fd4d85c448f50e9a428fe1a633b7b4a61300ea70e3639912880f0548725e +size 99648 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..870929356f93fe7c1cb828ff222bb6d284628f91 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc10baa17f32ef8c8245fef34531412776dceed4484b30851d1d39b2060c081d +size 50471 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..6cac3d9d79b0c24531a7b84d413c953bf3beb816 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 22.211621044921873, + "l0_per_token_0.001": 22.211621044921873, + "l0_per_token_0.01": 22.211621044921873, + "l0_per_token_0.05": 22.211621044921873, + "l0_per_token_0.1": 22.211621044921873, + "l0_per_token_0.5": 22.211618603515625, + "l0_per_token_1.0": 22.20188544921875, + "l1_per_token": 305.64404951472955, + "r2_score": 0.9158018148256896, + "l2_loss": 1.9746523668615905 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc5820d4738d1d2ea7d2973b3b80fe713cc65e69 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4f23398d36159792223ddda4d7de2210a05a6fa620b03cf798ffa06fe1d4037 +size 50382 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..756e45a8851a7819b1416f55866dd3013582505f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02acd6d5fe625002e3b5847ee69517c27ac90700d7c8cc1fa8f38014b45e2882 +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f459007eba0f10d0ddfaaa6fc715e4bdaf87c78c --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fad1d7894e17adca839d311ad9aa3c319ff5614df470537817dfe08042d53ae0 +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/config.json b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8465153a9e9a1f2031f87935b38abdc81b8f3232 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8cd4fe5407991c8a40b555133df223a666b6afa6 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:379bc9dc7109b106af2eae69956a5b08db6fa5b8f1aec8d5a08bce0e986bc977 +size 75550008 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..56bfc6af7ba65f03f48c69917917c8fb86cd43f9 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6eaaf3a285a327c928707e62d0f6b8ca4f7ff9ddb9d035c05e98cb5b9cc62326 +size 99648 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..924af6ca4e55610e2160087c39a3441cbf8aba0d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8935cd7f6f5d5cc135eb2e6d93cfe9be4399b424571ef87d27930531d16e6ef3 +size 50471 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b9cb9483d8ae7cb571d3c72810ed2e1022fa6f8a --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 21.72416220703125, + "l0_per_token_0.001": 21.72416220703125, + "l0_per_token_0.01": 21.72416220703125, + "l0_per_token_0.05": 21.72416220703125, + "l0_per_token_0.1": 21.72416220703125, + "l0_per_token_0.5": 21.72412216796875, + "l0_per_token_1.0": 21.71222099609375, + "l1_per_token": 303.8500377439475, + "r2_score": 0.9152347604405267, + "l2_loss": 1.987951172310699 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fb2b61f53302d0e2917f082af092271bf989a58 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:997f5098feca8e8c8981ed8459bbee97197237fc99401a19dfad67cc5ccd42ce +size 50382 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6277864477fb3376b4ef869e149d0724df56cb08 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93611b8c55be04fa48ae3054962ebd285ff1cc116759b38914b691e70be34654 +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8bcf2aa4d66b91de6257fe96503a42a541efd95 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14c1815de43d1e3b24b957e7edbb19dddb0f169807d20089451747502205e32b +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/config.json b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..859d3bae315c9ef5dc1a4ece37711d5e66c114f1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/model.safetensors b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..50e8da7541a7dfbd467cd97bad3462c3403a4f60 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bec93ca96aa06359b3d6bfd409464b4c7e92b2f8e4a21a6eab6183b44de9537a +size 75550008 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f878c1f3a4137a671b3447781df33d9df80b83b1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8d7876e613853f56f617c6a25ed9c48b3379acba40d132747b0ec841dcf024e +size 99648 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d20728a873319187a70146487dde99d7418e7c2 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:906cf378b5378e8ef0fe1f10c1fbdf71f40a086581ea05981d7fec50ccd200f9 +size 50471 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..91d6770ce23465b0bb5831ab558039a85a12769a --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 22.74853115234375, + "l0_per_token_0.001": 22.74853115234375, + "l0_per_token_0.01": 22.74853115234375, + "l0_per_token_0.05": 22.74853115234375, + "l0_per_token_0.1": 22.74853115234375, + "l0_per_token_0.5": 22.7485296875, + "l0_per_token_1.0": 22.738066552734374, + "l1_per_token": 306.17978527802273, + "r2_score": 0.916500919721207, + "l2_loss": 1.9582566555554617 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6dcdd3fb5490ed1ca4fc7aa942104b39152ef41e --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:912a50002a50f107d60675b2f7b10c2053c8844e1d90e29348a4178d12e7e05d +size 50382 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d563efa699a676d994e7c019a1b6b9478d92e88f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62fed37de42cbb5de8316bb5becd6d9a504bfb625c0673054ed531694d5ac88f +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..63b69b6431fc1e524b806dd0ebba16327ce65d52 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:234d20ef70331375ca38906e844db38e3181b69455bb908615bbbb81418a37f9 +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/code_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x16/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..bf52b6c10723807f407a81298560e2eb53195b24 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 261.21647, + "recon_r2": 0.9847891900024385, + "l1_per_token": 802.3347534700799 + }, + "seed1": { + "l0": 259.5063866666667, + "recon_r2": 0.9847295416325721, + "l1_per_token": 801.1025795754 + }, + "seed2": { + "l0": 261.92269666666664, + "recon_r2": 0.9848399666541547, + "l1_per_token": 801.1532248487391 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.4838865486807893, + "mcc_abs": 0.4838936963635519, + "act_iou": 0.07724485732967691, + "act_iou_n_pairs": 12288, + "input_r2": 0.986300313248124, + "matched_mcc_signed": { + "0.6": 0.5669469130603053, + "0.7": 0.5792228421668155, + "0.8": 0.587894549012978, + "0.9": 0.6093967881892141 + }, + "matched_mcc_abs": { + "0.6": 0.5641969852384354, + "0.7": 0.5777336329062613, + "0.8": 0.5873191858114533, + "0.9": 0.6092722746776401 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.48617790817253054, + "mcc_abs": 0.4861868387082062, + "act_iou": 0.07751851479623832, + "act_iou_n_pairs": 12288, + "input_r2": 0.9863939795370449, + "matched_mcc_signed": { + "0.6": 0.56719229332494, + "0.7": 0.5801780364736703, + "0.8": 0.588202794611136, + "0.9": 0.6097011834566746 + }, + "matched_mcc_abs": { + "0.6": 0.5652036450506829, + "0.7": 0.578832735521309, + "0.8": 0.5877368393819278, + "0.9": 0.6101164550144322 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.48413444284111223, + "mcc_abs": 0.48414145967380245, + "act_iou": 0.0766522527661015, + "act_iou_n_pairs": 12288, + "input_r2": 0.9863100790127616, + "matched_mcc_signed": { + "0.6": 0.5657559649728287, + "0.7": 0.5808013478311531, + "0.8": 0.589705540257335, + "0.9": 0.6124598467073248 + }, + "matched_mcc_abs": { + "0.6": 0.5631692931558949, + "0.7": 0.5801305375231778, + "0.8": 0.5899832379324904, + "0.9": 0.6124261146745023 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 260.88185111111113, + "l0_std": 1.2424233797807718, + "recon_r2_mean": 0.9847862327630551, + "recon_r2_std": 5.527187618002564e-05, + "l1_per_token_mean": 801.5301859647398, + "l1_per_token_std": 0.6972358918723081, + "input_r2_mean": 0.9863347905993102, + "input_r2_std": 5.1491167226673785e-05, + "mcc_signed_mean": 0.48473296656481074, + "mcc_signed_std": 0.0012574796483325594, + "mcc_abs_mean": 0.48474066491518686, + "mcc_abs_std": 0.0012585351183771539, + "matched_mcc_signed_per_threshold": { + "0.6": 0.5666317237860247, + "0.7": 0.5800674088238796, + "0.8": 0.5886009612938163, + "0.9": 0.6105192727844045 + }, + "matched_mcc_signed_band": 0.5864548416720313, + "matched_mcc_abs_per_threshold": { + "0.6": 0.5641899744816711, + "0.7": 0.5788989686502494, + "0.8": 0.5883464210419572, + "0.9": 0.6106049481221916 + }, + "matched_mcc_abs_band": 0.5855100780740172, + "act_iou_mean": 0.07713854163067224, + "act_iou_std": 0.00044280892843903615, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/dict_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x16/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..3d2c2e647b27d5182b239ed7a1c3c3fa75e2320a --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 7989, + "0.6": 5686, + "0.7": 3713, + "0.8": 2032, + "0.9": 716 + }, + "n_dup_clusters_signed": { + "0.5": 2725, + "0.6": 1982, + "0.7": 1329, + "0.8": 744, + "0.9": 260 + }, + "largest_dup_signed": { + "0.5": 69, + "0.6": 44, + "0.7": 26, + "0.8": 16, + "0.9": 10 + }, + "babel_mu1_signed": 0.6284380555152893, + "n_dup_abs": { + "0.5": 8023, + "0.6": 5750, + "0.7": 3769, + "0.8": 2052, + "0.9": 720 + }, + "n_dup_clusters_abs": { + "0.5": 2730, + "0.6": 2006, + "0.7": 1355, + "0.8": 754, + "0.9": 261 + }, + "largest_dup_abs": { + "0.5": 47, + "0.6": 36, + "0.7": 26, + "0.8": 16, + "0.9": 10 + }, + "babel_mu1_abs": 0.6307905316352844 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 7991, + "0.6": 5701, + "0.7": 3669, + "0.8": 2005, + "0.9": 736 + }, + "n_dup_clusters_signed": { + "0.5": 2712, + "0.6": 1994, + "0.7": 1296, + "0.8": 723, + "0.9": 256 + }, + "largest_dup_signed": { + "0.5": 50, + "0.6": 31, + "0.7": 29, + "0.8": 26, + "0.9": 10 + }, + "babel_mu1_signed": 0.6283772587776184, + "n_dup_abs": { + "0.5": 8014, + "0.6": 5768, + "0.7": 3727, + "0.8": 2023, + "0.9": 739 + }, + "n_dup_clusters_abs": { + "0.5": 2718, + "0.6": 2024, + "0.7": 1322, + "0.8": 731, + "0.9": 258 + }, + "largest_dup_abs": { + "0.5": 43, + "0.6": 32, + "0.7": 29, + "0.8": 26, + "0.9": 10 + }, + "babel_mu1_abs": 0.6305112242698669 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 7991, + "0.6": 5734, + "0.7": 3681, + "0.8": 2006, + "0.9": 722 + }, + "n_dup_clusters_signed": { + "0.5": 2738, + "0.6": 2027, + "0.7": 1318, + "0.8": 745, + "0.9": 270 + }, + "largest_dup_signed": { + "0.5": 54, + "0.6": 37, + "0.7": 22, + "0.8": 16, + "0.9": 11 + }, + "babel_mu1_signed": 0.6283814311027527, + "n_dup_abs": { + "0.5": 8024, + "0.6": 5789, + "0.7": 3734, + "0.8": 2021, + "0.9": 724 + }, + "n_dup_clusters_abs": { + "0.5": 2748, + "0.6": 2051, + "0.7": 1342, + "0.8": 751, + "0.9": 271 + }, + "largest_dup_abs": { + "0.5": 55, + "0.6": 40, + "0.7": 22, + "0.8": 16, + "0.9": 11 + }, + "babel_mu1_abs": 0.6304722428321838 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8436322212219238, + "dcs_abs": 0.8466837406158447, + "frac_sign_flipped": 0.021077473958333332, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8275504112243652, + "0.6": 0.8313969373703003, + "0.7": 0.8353726863861084, + "0.8": 0.8393656611442566, + "0.9": 0.8434174656867981 + }, + "unique_frac_signed": { + "0.5": 0.5710042317708334, + "0.6": 0.6984456380208334, + "0.7": 0.8064371744791666, + "0.8": 0.8954264322916666, + "0.9": 0.9619140625 + }, + "dedup_dcs_signed": { + "0.5": 0.4725347868128059, + "0.6": 0.5806855643701663, + "0.7": 0.6736755888462842, + "0.8": 0.7515901993465377, + "0.9": 0.8112951208022423 + }, + "matched_frac_signed": { + "0.5": 0.5017903645833334, + "0.6": 0.5933430989583334, + "0.7": 0.641845703125, + "0.8": 0.6387532552083334, + "0.9": 0.56201171875 + }, + "matched_cos_abs": { + "0.5": 0.8312228918075562, + "0.6": 0.8351380825042725, + "0.7": 0.8387950658798218, + "0.8": 0.8426327705383301, + "0.9": 0.8464468717575073 + }, + "unique_frac_abs": { + "0.5": 0.5691324869791666, + "0.6": 0.6953125, + "0.7": 0.8039143880208334, + "0.8": 0.8946126302083334, + "0.9": 0.9617513020833334 + }, + "dedup_dcs_abs": { + "0.5": 0.4730759516484492, + "0.6": 0.580681947991252, + "0.7": 0.6743194220616715, + "0.8": 0.7538299191510305, + "0.9": 0.8140713810571469 + }, + "matched_frac_abs": { + "0.5": 0.502685546875, + "0.6": 0.5953776041666666, + "0.7": 0.6446126302083334, + "0.8": 0.6398111979166666, + "0.9": 0.56201171875 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8473139405250549, + "dcs_abs": 0.8499342799186707, + "frac_sign_flipped": 0.019205729166666668, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8293575048446655, + "0.6": 0.8332625031471252, + "0.7": 0.8369428515434265, + "0.8": 0.8419058322906494, + "0.9": 0.845782995223999 + }, + "unique_frac_signed": { + "0.5": 0.5720621744791666, + "0.6": 0.6984456380208334, + "0.7": 0.8068440755208334, + "0.8": 0.8962809244791666, + "0.9": 0.9630533854166666 + }, + "dedup_dcs_signed": { + "0.5": 0.47444405764205533, + "0.6": 0.5819885606494306, + "0.7": 0.6752823813173261, + "0.8": 0.7545841376898654, + "0.9": 0.8145341768783206 + }, + "matched_frac_signed": { + "0.5": 0.5034993489583334, + "0.6": 0.595458984375, + "0.7": 0.642822265625, + "0.8": 0.6434733072916666, + "0.9": 0.5736490885416666 + }, + "matched_cos_abs": { + "0.5": 0.8338408470153809, + "0.6": 0.8370147943496704, + "0.7": 0.8400601744651794, + "0.8": 0.8446245789527893, + "0.9": 0.8484957814216614 + }, + "unique_frac_abs": { + "0.5": 0.5699462890625, + "0.6": 0.695556640625, + "0.7": 0.804443359375, + "0.8": 0.8955078125, + "0.9": 0.962890625 + }, + "dedup_dcs_abs": { + "0.5": 0.4752444964251481, + "0.6": 0.582191198511282, + "0.7": 0.6757808288239175, + "0.8": 0.7563679090817459, + "0.9": 0.8170086332829669 + }, + "matched_frac_abs": { + "0.5": 0.5048014322916666, + "0.6": 0.5970865885416666, + "0.7": 0.6463216145833334, + "0.8": 0.6448567708333334, + "0.9": 0.5731608072916666 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8424146771430969, + "dcs_abs": 0.8454044461250305, + "frac_sign_flipped": 0.020263671875, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8259482383728027, + "0.6": 0.8288151025772095, + "0.7": 0.8321582078933716, + "0.8": 0.8369964957237244, + "0.9": 0.8409832119941711 + }, + "unique_frac_signed": { + "0.5": 0.5714518229166666, + "0.6": 0.6983235677083334, + "0.7": 0.8072916666666666, + "0.8": 0.8965250651041666, + "0.9": 0.9620768229166666 + }, + "dedup_dcs_signed": { + "0.5": 0.4719896264529476, + "0.6": 0.5787811194022652, + "0.7": 0.6717943865805864, + "0.8": 0.7503883378206713, + "0.9": 0.8090904567216057 + }, + "matched_frac_signed": { + "0.5": 0.4991048177083333, + "0.6": 0.5921223958333334, + "0.7": 0.6378580729166666, + "0.8": 0.6355794270833334, + "0.9": 0.555908203125 + }, + "matched_cos_abs": { + "0.5": 0.8300459980964661, + "0.6": 0.8329750299453735, + "0.7": 0.8358787894248962, + "0.8": 0.8401848077774048, + "0.9": 0.84405916929245 + }, + "unique_frac_abs": { + "0.5": 0.56982421875, + "0.6": 0.695556640625, + "0.7": 0.8048095703125, + "0.8": 0.895751953125, + "0.9": 0.9619954427083334 + }, + "dedup_dcs_abs": { + "0.5": 0.47298031239188276, + "0.6": 0.5793813135533128, + "0.7": 0.6727232493503834, + "0.8": 0.752597182552563, + "0.9": 0.8119810742355185 + }, + "matched_frac_abs": { + "0.5": 0.5013834635416666, + "0.6": 0.5930989583333334, + "0.7": 0.6405436197916666, + "0.8": 0.6357421875, + "0.9": 0.5555826822916666 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.6283989151318868, + "babel_mu1_signed_std": 3.396070205293077e-05, + "n_dup_signed_per_threshold": { + "0.5": 7990.333333333333, + "0.6": 5707.0, + "0.7": 3687.6666666666665, + "0.8": 2014.3333333333333, + "0.9": 724.6666666666666 + }, + "n_dup_signed_band": 3033.4166666666665, + "n_dup_signed_band_std": 4.932882862316248, + "n_dup_clusters_signed_per_threshold": { + "0.5": 2725.0, + "0.6": 2001.0, + "0.7": 1314.3333333333333, + "0.8": 737.3333333333334, + "0.9": 262.0 + }, + "n_dup_clusters_signed_band": 1078.6666666666665, + "n_dup_clusters_signed_band_std": 11.375228935425138, + "largest_dup_signed_per_threshold": { + "0.5": 57.666666666666664, + "0.6": 37.333333333333336, + "0.7": 25.666666666666668, + "0.8": 19.333333333333332, + "0.9": 10.333333333333334 + }, + "largest_dup_signed_band": 23.166666666666664, + "largest_dup_signed_band_std": 1.4433756729740643, + "dcs_signed_mean": 0.8444536129633585, + "dcs_signed_std": 0.0025508251049653602, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.4729894903026029, + "0.6": 0.5804850814739541, + "0.7": 0.6735841189147322, + "0.8": 0.7521875582856915, + "0.9": 0.8116399181340562 + }, + "dedup_dcs_signed_band": 0.7044741692021086, + "dedup_dcs_signed_band_std": 0.0020467164304809188, + "matched_cos_signed_per_threshold": { + "0.5": 0.8276187181472778, + "0.6": 0.831158181031545, + "0.7": 0.8348245819409689, + "0.8": 0.8394226630528768, + "0.9": 0.8433945576349894 + }, + "matched_cos_signed_band": 0.837199995915095, + "matched_cos_signed_band_std": 0.0023732482633511687, + "unique_frac_signed_per_threshold": { + "0.5": 0.5715060763888888, + "0.6": 0.6984049479166666, + "0.7": 0.8068576388888888, + "0.8": 0.8960774739583334, + "0.9": 0.9623480902777777 + }, + "unique_frac_signed_band": 0.8409220377604166, + "unique_frac_signed_band_std": 0.0003212006307748867, + "matched_frac_signed_per_threshold": { + "0.5": 0.50146484375, + "0.6": 0.5936414930555557, + "0.7": 0.6408420138888888, + "0.8": 0.6392686631944445, + "0.9": 0.5638563368055555 + }, + "matched_frac_signed_band": 0.6094021267361112, + "matched_frac_signed_band_std": 0.004257045162116399, + "babel_mu1_abs_mean": 0.6305913329124451, + "babel_mu1_abs_std": 0.00017360871667800333, + "n_dup_abs_per_threshold": { + "0.5": 8020.333333333333, + "0.6": 5769.0, + "0.7": 3743.3333333333335, + "0.8": 2032.0, + "0.9": 727.6666666666666 + }, + "n_dup_abs_band": 3068.0, + "n_dup_abs_band_std": 4.337337893224368, + "n_dup_clusters_abs_per_threshold": { + "0.5": 2732.0, + "0.6": 2027.0, + "0.7": 1339.6666666666667, + "0.8": 745.3333333333334, + "0.9": 263.3333333333333 + }, + "n_dup_clusters_abs_band": 1093.8333333333333, + "n_dup_clusters_abs_band_std": 10.001041612418845, + "largest_dup_abs_per_threshold": { + "0.5": 48.333333333333336, + "0.6": 36.0, + "0.7": 25.666666666666668, + "0.8": 19.333333333333332, + "0.9": 10.333333333333334 + }, + "largest_dup_abs_band": 22.833333333333332, + "largest_dup_abs_band_std": 1.2332207155790618, + "dcs_abs_mean": 0.8473408222198486, + "dcs_abs_std": 0.0023353084860979947, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.47376692015516, + "0.6": 0.5807514866852822, + "0.7": 0.6742745000786575, + "0.8": 0.7542650035951132, + "0.9": 0.8143536961918775 + }, + "dedup_dcs_abs_band": 0.7059111716377326, + "dedup_dcs_abs_band_std": 0.0018402444972756755, + "matched_cos_abs_per_threshold": { + "0.5": 0.831703245639801, + "0.6": 0.8350426355997721, + "0.7": 0.8382446765899658, + "0.8": 0.8424807190895081, + "0.9": 0.8463339408238729 + }, + "matched_cos_abs_band": 0.8405254930257797, + "matched_cos_abs_band_std": 0.002146270011368372, + "unique_frac_abs_per_threshold": { + "0.5": 0.5696343315972222, + "0.6": 0.6954752604166666, + "0.7": 0.8043891059027778, + "0.8": 0.8952907986111112, + "0.9": 0.9622124565972223 + }, + "unique_frac_abs_band": 0.8393419053819444, + "unique_frac_abs_band_std": 0.0003863328434643635, + "matched_frac_abs_per_threshold": { + "0.5": 0.502956814236111, + "0.6": 0.5951877170138888, + "0.7": 0.6438259548611112, + "0.8": 0.64013671875, + "0.9": 0.5635850694444443 + }, + "matched_frac_abs_band": 0.610683865017361, + "matched_frac_abs_band_std": 0.004561664360585359, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/config.json b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..582c8dce741510337aae6bc33e8db9b021f25bc1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/model.safetensors b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..39760dbba21ca823364c989ae4c76afe3d6a32a1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2e1242727f192d221c9f5b036a3c7667c7fd6ff55c6f2bb290d5588852aa57c +size 75550008 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..302a103cec575f786c765105d88256ee3c0cd1ad --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a22fd4d85c448f50e9a428fe1a633b7b4a61300ea70e3639912880f0548725e +size 99648 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..870929356f93fe7c1cb828ff222bb6d284628f91 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc10baa17f32ef8c8245fef34531412776dceed4484b30851d1d39b2060c081d +size 50471 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..6cac3d9d79b0c24531a7b84d413c953bf3beb816 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 22.211621044921873, + "l0_per_token_0.001": 22.211621044921873, + "l0_per_token_0.01": 22.211621044921873, + "l0_per_token_0.05": 22.211621044921873, + "l0_per_token_0.1": 22.211621044921873, + "l0_per_token_0.5": 22.211618603515625, + "l0_per_token_1.0": 22.20188544921875, + "l1_per_token": 305.64404951472955, + "r2_score": 0.9158018148256896, + "l2_loss": 1.9746523668615905 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc5820d4738d1d2ea7d2973b3b80fe713cc65e69 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4f23398d36159792223ddda4d7de2210a05a6fa620b03cf798ffa06fe1d4037 +size 50382 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..756e45a8851a7819b1416f55866dd3013582505f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02acd6d5fe625002e3b5847ee69517c27ac90700d7c8cc1fa8f38014b45e2882 +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f459007eba0f10d0ddfaaa6fc715e4bdaf87c78c --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fad1d7894e17adca839d311ad9aa3c319ff5614df470537817dfe08042d53ae0 +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/config.json b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8465153a9e9a1f2031f87935b38abdc81b8f3232 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/model.safetensors b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8cd4fe5407991c8a40b555133df223a666b6afa6 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:379bc9dc7109b106af2eae69956a5b08db6fa5b8f1aec8d5a08bce0e986bc977 +size 75550008 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..56bfc6af7ba65f03f48c69917917c8fb86cd43f9 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6eaaf3a285a327c928707e62d0f6b8ca4f7ff9ddb9d035c05e98cb5b9cc62326 +size 99648 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..924af6ca4e55610e2160087c39a3441cbf8aba0d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8935cd7f6f5d5cc135eb2e6d93cfe9be4399b424571ef87d27930531d16e6ef3 +size 50471 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b9cb9483d8ae7cb571d3c72810ed2e1022fa6f8a --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 21.72416220703125, + "l0_per_token_0.001": 21.72416220703125, + "l0_per_token_0.01": 21.72416220703125, + "l0_per_token_0.05": 21.72416220703125, + "l0_per_token_0.1": 21.72416220703125, + "l0_per_token_0.5": 21.72412216796875, + "l0_per_token_1.0": 21.71222099609375, + "l1_per_token": 303.8500377439475, + "r2_score": 0.9152347604405267, + "l2_loss": 1.987951172310699 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fb2b61f53302d0e2917f082af092271bf989a58 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:997f5098feca8e8c8981ed8459bbee97197237fc99401a19dfad67cc5ccd42ce +size 50382 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6277864477fb3376b4ef869e149d0724df56cb08 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93611b8c55be04fa48ae3054962ebd285ff1cc116759b38914b691e70be34654 +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8bcf2aa4d66b91de6257fe96503a42a541efd95 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14c1815de43d1e3b24b957e7edbb19dddb0f169807d20089451747502205e32b +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/config.json b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..859d3bae315c9ef5dc1a4ece37711d5e66c114f1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/model.safetensors b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..50e8da7541a7dfbd467cd97bad3462c3403a4f60 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bec93ca96aa06359b3d6bfd409464b4c7e92b2f8e4a21a6eab6183b44de9537a +size 75550008 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f878c1f3a4137a671b3447781df33d9df80b83b1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8d7876e613853f56f617c6a25ed9c48b3379acba40d132747b0ec841dcf024e +size 99648 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d20728a873319187a70146487dde99d7418e7c2 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:906cf378b5378e8ef0fe1f10c1fbdf71f40a086581ea05981d7fec50ccd200f9 +size 50471 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..91d6770ce23465b0bb5831ab558039a85a12769a --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 22.74853115234375, + "l0_per_token_0.001": 22.74853115234375, + "l0_per_token_0.01": 22.74853115234375, + "l0_per_token_0.05": 22.74853115234375, + "l0_per_token_0.1": 22.74853115234375, + "l0_per_token_0.5": 22.7485296875, + "l0_per_token_1.0": 22.738066552734374, + "l1_per_token": 306.17978527802273, + "r2_score": 0.916500919721207, + "l2_loss": 1.9582566555554617 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6dcdd3fb5490ed1ca4fc7aa942104b39152ef41e --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:912a50002a50f107d60675b2f7b10c2053c8844e1d90e29348a4178d12e7e05d +size 50382 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d563efa699a676d994e7c019a1b6b9478d92e88f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62fed37de42cbb5de8316bb5becd6d9a504bfb625c0673054ed531694d5ac88f +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..63b69b6431fc1e524b806dd0ebba16327ce65d52 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:234d20ef70331375ca38906e844db38e3181b69455bb908615bbbb81418a37f9 +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/code_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x16/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..bf52b6c10723807f407a81298560e2eb53195b24 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 261.21647, + "recon_r2": 0.9847891900024385, + "l1_per_token": 802.3347534700799 + }, + "seed1": { + "l0": 259.5063866666667, + "recon_r2": 0.9847295416325721, + "l1_per_token": 801.1025795754 + }, + "seed2": { + "l0": 261.92269666666664, + "recon_r2": 0.9848399666541547, + "l1_per_token": 801.1532248487391 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.4838865486807893, + "mcc_abs": 0.4838936963635519, + "act_iou": 0.07724485732967691, + "act_iou_n_pairs": 12288, + "input_r2": 0.986300313248124, + "matched_mcc_signed": { + "0.6": 0.5669469130603053, + "0.7": 0.5792228421668155, + "0.8": 0.587894549012978, + "0.9": 0.6093967881892141 + }, + "matched_mcc_abs": { + "0.6": 0.5641969852384354, + "0.7": 0.5777336329062613, + "0.8": 0.5873191858114533, + "0.9": 0.6092722746776401 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.48617790817253054, + "mcc_abs": 0.4861868387082062, + "act_iou": 0.07751851479623832, + "act_iou_n_pairs": 12288, + "input_r2": 0.9863939795370449, + "matched_mcc_signed": { + "0.6": 0.56719229332494, + "0.7": 0.5801780364736703, + "0.8": 0.588202794611136, + "0.9": 0.6097011834566746 + }, + "matched_mcc_abs": { + "0.6": 0.5652036450506829, + "0.7": 0.578832735521309, + "0.8": 0.5877368393819278, + "0.9": 0.6101164550144322 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.48413444284111223, + "mcc_abs": 0.48414145967380245, + "act_iou": 0.0766522527661015, + "act_iou_n_pairs": 12288, + "input_r2": 0.9863100790127616, + "matched_mcc_signed": { + "0.6": 0.5657559649728287, + "0.7": 0.5808013478311531, + "0.8": 0.589705540257335, + "0.9": 0.6124598467073248 + }, + "matched_mcc_abs": { + "0.6": 0.5631692931558949, + "0.7": 0.5801305375231778, + "0.8": 0.5899832379324904, + "0.9": 0.6124261146745023 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 260.88185111111113, + "l0_std": 1.2424233797807718, + "recon_r2_mean": 0.9847862327630551, + "recon_r2_std": 5.527187618002564e-05, + "l1_per_token_mean": 801.5301859647398, + "l1_per_token_std": 0.6972358918723081, + "input_r2_mean": 0.9863347905993102, + "input_r2_std": 5.1491167226673785e-05, + "mcc_signed_mean": 0.48473296656481074, + "mcc_signed_std": 0.0012574796483325594, + "mcc_abs_mean": 0.48474066491518686, + "mcc_abs_std": 0.0012585351183771539, + "matched_mcc_signed_per_threshold": { + "0.6": 0.5666317237860247, + "0.7": 0.5800674088238796, + "0.8": 0.5886009612938163, + "0.9": 0.6105192727844045 + }, + "matched_mcc_signed_band": 0.5864548416720313, + "matched_mcc_abs_per_threshold": { + "0.6": 0.5641899744816711, + "0.7": 0.5788989686502494, + "0.8": 0.5883464210419572, + "0.9": 0.6106049481221916 + }, + "matched_mcc_abs_band": 0.5855100780740172, + "act_iou_mean": 0.07713854163067224, + "act_iou_std": 0.00044280892843903615, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/dict_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x16/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..3d2c2e647b27d5182b239ed7a1c3c3fa75e2320a --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 7989, + "0.6": 5686, + "0.7": 3713, + "0.8": 2032, + "0.9": 716 + }, + "n_dup_clusters_signed": { + "0.5": 2725, + "0.6": 1982, + "0.7": 1329, + "0.8": 744, + "0.9": 260 + }, + "largest_dup_signed": { + "0.5": 69, + "0.6": 44, + "0.7": 26, + "0.8": 16, + "0.9": 10 + }, + "babel_mu1_signed": 0.6284380555152893, + "n_dup_abs": { + "0.5": 8023, + "0.6": 5750, + "0.7": 3769, + "0.8": 2052, + "0.9": 720 + }, + "n_dup_clusters_abs": { + "0.5": 2730, + "0.6": 2006, + "0.7": 1355, + "0.8": 754, + "0.9": 261 + }, + "largest_dup_abs": { + "0.5": 47, + "0.6": 36, + "0.7": 26, + "0.8": 16, + "0.9": 10 + }, + "babel_mu1_abs": 0.6307905316352844 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 7991, + "0.6": 5701, + "0.7": 3669, + "0.8": 2005, + "0.9": 736 + }, + "n_dup_clusters_signed": { + "0.5": 2712, + "0.6": 1994, + "0.7": 1296, + "0.8": 723, + "0.9": 256 + }, + "largest_dup_signed": { + "0.5": 50, + "0.6": 31, + "0.7": 29, + "0.8": 26, + "0.9": 10 + }, + "babel_mu1_signed": 0.6283772587776184, + "n_dup_abs": { + "0.5": 8014, + "0.6": 5768, + "0.7": 3727, + "0.8": 2023, + "0.9": 739 + }, + "n_dup_clusters_abs": { + "0.5": 2718, + "0.6": 2024, + "0.7": 1322, + "0.8": 731, + "0.9": 258 + }, + "largest_dup_abs": { + "0.5": 43, + "0.6": 32, + "0.7": 29, + "0.8": 26, + "0.9": 10 + }, + "babel_mu1_abs": 0.6305112242698669 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 7991, + "0.6": 5734, + "0.7": 3681, + "0.8": 2006, + "0.9": 722 + }, + "n_dup_clusters_signed": { + "0.5": 2738, + "0.6": 2027, + "0.7": 1318, + "0.8": 745, + "0.9": 270 + }, + "largest_dup_signed": { + "0.5": 54, + "0.6": 37, + "0.7": 22, + "0.8": 16, + "0.9": 11 + }, + "babel_mu1_signed": 0.6283814311027527, + "n_dup_abs": { + "0.5": 8024, + "0.6": 5789, + "0.7": 3734, + "0.8": 2021, + "0.9": 724 + }, + "n_dup_clusters_abs": { + "0.5": 2748, + "0.6": 2051, + "0.7": 1342, + "0.8": 751, + "0.9": 271 + }, + "largest_dup_abs": { + "0.5": 55, + "0.6": 40, + "0.7": 22, + "0.8": 16, + "0.9": 11 + }, + "babel_mu1_abs": 0.6304722428321838 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8436322212219238, + "dcs_abs": 0.8466837406158447, + "frac_sign_flipped": 0.021077473958333332, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8275504112243652, + "0.6": 0.8313969373703003, + "0.7": 0.8353726863861084, + "0.8": 0.8393656611442566, + "0.9": 0.8434174656867981 + }, + "unique_frac_signed": { + "0.5": 0.5710042317708334, + "0.6": 0.6984456380208334, + "0.7": 0.8064371744791666, + "0.8": 0.8954264322916666, + "0.9": 0.9619140625 + }, + "dedup_dcs_signed": { + "0.5": 0.4725347868128059, + "0.6": 0.5806855643701663, + "0.7": 0.6736755888462842, + "0.8": 0.7515901993465377, + "0.9": 0.8112951208022423 + }, + "matched_frac_signed": { + "0.5": 0.5017903645833334, + "0.6": 0.5933430989583334, + "0.7": 0.641845703125, + "0.8": 0.6387532552083334, + "0.9": 0.56201171875 + }, + "matched_cos_abs": { + "0.5": 0.8312228918075562, + "0.6": 0.8351380825042725, + "0.7": 0.8387950658798218, + "0.8": 0.8426327705383301, + "0.9": 0.8464468717575073 + }, + "unique_frac_abs": { + "0.5": 0.5691324869791666, + "0.6": 0.6953125, + "0.7": 0.8039143880208334, + "0.8": 0.8946126302083334, + "0.9": 0.9617513020833334 + }, + "dedup_dcs_abs": { + "0.5": 0.4730759516484492, + "0.6": 0.580681947991252, + "0.7": 0.6743194220616715, + "0.8": 0.7538299191510305, + "0.9": 0.8140713810571469 + }, + "matched_frac_abs": { + "0.5": 0.502685546875, + "0.6": 0.5953776041666666, + "0.7": 0.6446126302083334, + "0.8": 0.6398111979166666, + "0.9": 0.56201171875 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8473139405250549, + "dcs_abs": 0.8499342799186707, + "frac_sign_flipped": 0.019205729166666668, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8293575048446655, + "0.6": 0.8332625031471252, + "0.7": 0.8369428515434265, + "0.8": 0.8419058322906494, + "0.9": 0.845782995223999 + }, + "unique_frac_signed": { + "0.5": 0.5720621744791666, + "0.6": 0.6984456380208334, + "0.7": 0.8068440755208334, + "0.8": 0.8962809244791666, + "0.9": 0.9630533854166666 + }, + "dedup_dcs_signed": { + "0.5": 0.47444405764205533, + "0.6": 0.5819885606494306, + "0.7": 0.6752823813173261, + "0.8": 0.7545841376898654, + "0.9": 0.8145341768783206 + }, + "matched_frac_signed": { + "0.5": 0.5034993489583334, + "0.6": 0.595458984375, + "0.7": 0.642822265625, + "0.8": 0.6434733072916666, + "0.9": 0.5736490885416666 + }, + "matched_cos_abs": { + "0.5": 0.8338408470153809, + "0.6": 0.8370147943496704, + "0.7": 0.8400601744651794, + "0.8": 0.8446245789527893, + "0.9": 0.8484957814216614 + }, + "unique_frac_abs": { + "0.5": 0.5699462890625, + "0.6": 0.695556640625, + "0.7": 0.804443359375, + "0.8": 0.8955078125, + "0.9": 0.962890625 + }, + "dedup_dcs_abs": { + "0.5": 0.4752444964251481, + "0.6": 0.582191198511282, + "0.7": 0.6757808288239175, + "0.8": 0.7563679090817459, + "0.9": 0.8170086332829669 + }, + "matched_frac_abs": { + "0.5": 0.5048014322916666, + "0.6": 0.5970865885416666, + "0.7": 0.6463216145833334, + "0.8": 0.6448567708333334, + "0.9": 0.5731608072916666 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8424146771430969, + "dcs_abs": 0.8454044461250305, + "frac_sign_flipped": 0.020263671875, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8259482383728027, + "0.6": 0.8288151025772095, + "0.7": 0.8321582078933716, + "0.8": 0.8369964957237244, + "0.9": 0.8409832119941711 + }, + "unique_frac_signed": { + "0.5": 0.5714518229166666, + "0.6": 0.6983235677083334, + "0.7": 0.8072916666666666, + "0.8": 0.8965250651041666, + "0.9": 0.9620768229166666 + }, + "dedup_dcs_signed": { + "0.5": 0.4719896264529476, + "0.6": 0.5787811194022652, + "0.7": 0.6717943865805864, + "0.8": 0.7503883378206713, + "0.9": 0.8090904567216057 + }, + "matched_frac_signed": { + "0.5": 0.4991048177083333, + "0.6": 0.5921223958333334, + "0.7": 0.6378580729166666, + "0.8": 0.6355794270833334, + "0.9": 0.555908203125 + }, + "matched_cos_abs": { + "0.5": 0.8300459980964661, + "0.6": 0.8329750299453735, + "0.7": 0.8358787894248962, + "0.8": 0.8401848077774048, + "0.9": 0.84405916929245 + }, + "unique_frac_abs": { + "0.5": 0.56982421875, + "0.6": 0.695556640625, + "0.7": 0.8048095703125, + "0.8": 0.895751953125, + "0.9": 0.9619954427083334 + }, + "dedup_dcs_abs": { + "0.5": 0.47298031239188276, + "0.6": 0.5793813135533128, + "0.7": 0.6727232493503834, + "0.8": 0.752597182552563, + "0.9": 0.8119810742355185 + }, + "matched_frac_abs": { + "0.5": 0.5013834635416666, + "0.6": 0.5930989583333334, + "0.7": 0.6405436197916666, + "0.8": 0.6357421875, + "0.9": 0.5555826822916666 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.6283989151318868, + "babel_mu1_signed_std": 3.396070205293077e-05, + "n_dup_signed_per_threshold": { + "0.5": 7990.333333333333, + "0.6": 5707.0, + "0.7": 3687.6666666666665, + "0.8": 2014.3333333333333, + "0.9": 724.6666666666666 + }, + "n_dup_signed_band": 3033.4166666666665, + "n_dup_signed_band_std": 4.932882862316248, + "n_dup_clusters_signed_per_threshold": { + "0.5": 2725.0, + "0.6": 2001.0, + "0.7": 1314.3333333333333, + "0.8": 737.3333333333334, + "0.9": 262.0 + }, + "n_dup_clusters_signed_band": 1078.6666666666665, + "n_dup_clusters_signed_band_std": 11.375228935425138, + "largest_dup_signed_per_threshold": { + "0.5": 57.666666666666664, + "0.6": 37.333333333333336, + "0.7": 25.666666666666668, + "0.8": 19.333333333333332, + "0.9": 10.333333333333334 + }, + "largest_dup_signed_band": 23.166666666666664, + "largest_dup_signed_band_std": 1.4433756729740643, + "dcs_signed_mean": 0.8444536129633585, + "dcs_signed_std": 0.0025508251049653602, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.4729894903026029, + "0.6": 0.5804850814739541, + "0.7": 0.6735841189147322, + "0.8": 0.7521875582856915, + "0.9": 0.8116399181340562 + }, + "dedup_dcs_signed_band": 0.7044741692021086, + "dedup_dcs_signed_band_std": 0.0020467164304809188, + "matched_cos_signed_per_threshold": { + "0.5": 0.8276187181472778, + "0.6": 0.831158181031545, + "0.7": 0.8348245819409689, + "0.8": 0.8394226630528768, + "0.9": 0.8433945576349894 + }, + "matched_cos_signed_band": 0.837199995915095, + "matched_cos_signed_band_std": 0.0023732482633511687, + "unique_frac_signed_per_threshold": { + "0.5": 0.5715060763888888, + "0.6": 0.6984049479166666, + "0.7": 0.8068576388888888, + "0.8": 0.8960774739583334, + "0.9": 0.9623480902777777 + }, + "unique_frac_signed_band": 0.8409220377604166, + "unique_frac_signed_band_std": 0.0003212006307748867, + "matched_frac_signed_per_threshold": { + "0.5": 0.50146484375, + "0.6": 0.5936414930555557, + "0.7": 0.6408420138888888, + "0.8": 0.6392686631944445, + "0.9": 0.5638563368055555 + }, + "matched_frac_signed_band": 0.6094021267361112, + "matched_frac_signed_band_std": 0.004257045162116399, + "babel_mu1_abs_mean": 0.6305913329124451, + "babel_mu1_abs_std": 0.00017360871667800333, + "n_dup_abs_per_threshold": { + "0.5": 8020.333333333333, + "0.6": 5769.0, + "0.7": 3743.3333333333335, + "0.8": 2032.0, + "0.9": 727.6666666666666 + }, + "n_dup_abs_band": 3068.0, + "n_dup_abs_band_std": 4.337337893224368, + "n_dup_clusters_abs_per_threshold": { + "0.5": 2732.0, + "0.6": 2027.0, + "0.7": 1339.6666666666667, + "0.8": 745.3333333333334, + "0.9": 263.3333333333333 + }, + "n_dup_clusters_abs_band": 1093.8333333333333, + "n_dup_clusters_abs_band_std": 10.001041612418845, + "largest_dup_abs_per_threshold": { + "0.5": 48.333333333333336, + "0.6": 36.0, + "0.7": 25.666666666666668, + "0.8": 19.333333333333332, + "0.9": 10.333333333333334 + }, + "largest_dup_abs_band": 22.833333333333332, + "largest_dup_abs_band_std": 1.2332207155790618, + "dcs_abs_mean": 0.8473408222198486, + "dcs_abs_std": 0.0023353084860979947, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.47376692015516, + "0.6": 0.5807514866852822, + "0.7": 0.6742745000786575, + "0.8": 0.7542650035951132, + "0.9": 0.8143536961918775 + }, + "dedup_dcs_abs_band": 0.7059111716377326, + "dedup_dcs_abs_band_std": 0.0018402444972756755, + "matched_cos_abs_per_threshold": { + "0.5": 0.831703245639801, + "0.6": 0.8350426355997721, + "0.7": 0.8382446765899658, + "0.8": 0.8424807190895081, + "0.9": 0.8463339408238729 + }, + "matched_cos_abs_band": 0.8405254930257797, + "matched_cos_abs_band_std": 0.002146270011368372, + "unique_frac_abs_per_threshold": { + "0.5": 0.5696343315972222, + "0.6": 0.6954752604166666, + "0.7": 0.8043891059027778, + "0.8": 0.8952907986111112, + "0.9": 0.9622124565972223 + }, + "unique_frac_abs_band": 0.8393419053819444, + "unique_frac_abs_band_std": 0.0003863328434643635, + "matched_frac_abs_per_threshold": { + "0.5": 0.502956814236111, + "0.6": 0.5951877170138888, + "0.7": 0.6438259548611112, + "0.8": 0.64013671875, + "0.9": 0.5635850694444443 + }, + "matched_frac_abs_band": 0.610683865017361, + "matched_frac_abs_band_std": 0.004561664360585359, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/config.json b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..582c8dce741510337aae6bc33e8db9b021f25bc1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/model.safetensors b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..39760dbba21ca823364c989ae4c76afe3d6a32a1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2e1242727f192d221c9f5b036a3c7667c7fd6ff55c6f2bb290d5588852aa57c +size 75550008 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..302a103cec575f786c765105d88256ee3c0cd1ad --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a22fd4d85c448f50e9a428fe1a633b7b4a61300ea70e3639912880f0548725e +size 99648 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..870929356f93fe7c1cb828ff222bb6d284628f91 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc10baa17f32ef8c8245fef34531412776dceed4484b30851d1d39b2060c081d +size 50471 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..6cac3d9d79b0c24531a7b84d413c953bf3beb816 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 22.211621044921873, + "l0_per_token_0.001": 22.211621044921873, + "l0_per_token_0.01": 22.211621044921873, + "l0_per_token_0.05": 22.211621044921873, + "l0_per_token_0.1": 22.211621044921873, + "l0_per_token_0.5": 22.211618603515625, + "l0_per_token_1.0": 22.20188544921875, + "l1_per_token": 305.64404951472955, + "r2_score": 0.9158018148256896, + "l2_loss": 1.9746523668615905 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc5820d4738d1d2ea7d2973b3b80fe713cc65e69 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4f23398d36159792223ddda4d7de2210a05a6fa620b03cf798ffa06fe1d4037 +size 50382 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..756e45a8851a7819b1416f55866dd3013582505f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02acd6d5fe625002e3b5847ee69517c27ac90700d7c8cc1fa8f38014b45e2882 +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f459007eba0f10d0ddfaaa6fc715e4bdaf87c78c --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fad1d7894e17adca839d311ad9aa3c319ff5614df470537817dfe08042d53ae0 +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/config.json b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8465153a9e9a1f2031f87935b38abdc81b8f3232 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/model.safetensors b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8cd4fe5407991c8a40b555133df223a666b6afa6 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:379bc9dc7109b106af2eae69956a5b08db6fa5b8f1aec8d5a08bce0e986bc977 +size 75550008 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..56bfc6af7ba65f03f48c69917917c8fb86cd43f9 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6eaaf3a285a327c928707e62d0f6b8ca4f7ff9ddb9d035c05e98cb5b9cc62326 +size 99648 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..924af6ca4e55610e2160087c39a3441cbf8aba0d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8935cd7f6f5d5cc135eb2e6d93cfe9be4399b424571ef87d27930531d16e6ef3 +size 50471 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b9cb9483d8ae7cb571d3c72810ed2e1022fa6f8a --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 21.72416220703125, + "l0_per_token_0.001": 21.72416220703125, + "l0_per_token_0.01": 21.72416220703125, + "l0_per_token_0.05": 21.72416220703125, + "l0_per_token_0.1": 21.72416220703125, + "l0_per_token_0.5": 21.72412216796875, + "l0_per_token_1.0": 21.71222099609375, + "l1_per_token": 303.8500377439475, + "r2_score": 0.9152347604405267, + "l2_loss": 1.987951172310699 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fb2b61f53302d0e2917f082af092271bf989a58 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:997f5098feca8e8c8981ed8459bbee97197237fc99401a19dfad67cc5ccd42ce +size 50382 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6277864477fb3376b4ef869e149d0724df56cb08 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93611b8c55be04fa48ae3054962ebd285ff1cc116759b38914b691e70be34654 +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8bcf2aa4d66b91de6257fe96503a42a541efd95 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14c1815de43d1e3b24b957e7edbb19dddb0f169807d20089451747502205e32b +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/config.json b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..859d3bae315c9ef5dc1a4ece37711d5e66c114f1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/model.safetensors b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..50e8da7541a7dfbd467cd97bad3462c3403a4f60 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bec93ca96aa06359b3d6bfd409464b4c7e92b2f8e4a21a6eab6183b44de9537a +size 75550008 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f878c1f3a4137a671b3447781df33d9df80b83b1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8d7876e613853f56f617c6a25ed9c48b3379acba40d132747b0ec841dcf024e +size 99648 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d20728a873319187a70146487dde99d7418e7c2 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:906cf378b5378e8ef0fe1f10c1fbdf71f40a086581ea05981d7fec50ccd200f9 +size 50471 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..91d6770ce23465b0bb5831ab558039a85a12769a --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 22.74853115234375, + "l0_per_token_0.001": 22.74853115234375, + "l0_per_token_0.01": 22.74853115234375, + "l0_per_token_0.05": 22.74853115234375, + "l0_per_token_0.1": 22.74853115234375, + "l0_per_token_0.5": 22.7485296875, + "l0_per_token_1.0": 22.738066552734374, + "l1_per_token": 306.17978527802273, + "r2_score": 0.916500919721207, + "l2_loss": 1.9582566555554617 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6dcdd3fb5490ed1ca4fc7aa942104b39152ef41e --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:912a50002a50f107d60675b2f7b10c2053c8844e1d90e29348a4178d12e7e05d +size 50382 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d563efa699a676d994e7c019a1b6b9478d92e88f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62fed37de42cbb5de8316bb5becd6d9a504bfb625c0673054ed531694d5ac88f +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..63b69b6431fc1e524b806dd0ebba16327ce65d52 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:234d20ef70331375ca38906e844db38e3181b69455bb908615bbbb81418a37f9 +size 50387 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/code_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x4/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..ba1efb88b6e7c7c96030648a3d292a2a27ae16b4 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 73.26211666666667, + "recon_r2": 0.9226123499770492, + "l1_per_token": 483.66945199742497 + }, + "seed1": { + "l0": 73.88305833333334, + "recon_r2": 0.9228508280066877, + "l1_per_token": 485.7527570608418 + }, + "seed2": { + "l0": 73.822995, + "recon_r2": 0.9229274013001743, + "l1_per_token": 485.3433463663552 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6457484252590913, + "mcc_abs": 0.6457676334652961, + "act_iou": 0.19626021556076378, + "act_iou_n_pairs": 3072, + "input_r2": 0.9566984295000585, + "matched_mcc_signed": { + "0.6": 0.7282515952041155, + "0.7": 0.7317495131780405, + "0.8": 0.7411449981760442, + "0.9": 0.7545275621589508 + }, + "matched_mcc_abs": { + "0.6": 0.7265693911147596, + "0.7": 0.7333936392047536, + "0.8": 0.7418744218329142, + "0.9": 0.7535220217329011 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6490924876488843, + "mcc_abs": 0.6491249224096499, + "act_iou": 0.19775168524089884, + "act_iou_n_pairs": 3072, + "input_r2": 0.9569147313292561, + "matched_mcc_signed": { + "0.6": 0.7281977294534634, + "0.7": 0.7341127887991568, + "0.8": 0.7396913838220901, + "0.9": 0.7516925504029341 + }, + "matched_mcc_abs": { + "0.6": 0.7293139891095975, + "0.7": 0.7349032563682367, + "0.8": 0.7400748328155513, + "0.9": 0.7515885447899787 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6388531379432676, + "mcc_abs": 0.6388793156724867, + "act_iou": 0.19276400938233987, + "act_iou_n_pairs": 3072, + "input_r2": 0.9561670172347472, + "matched_mcc_signed": { + "0.6": 0.7210126255808897, + "0.7": 0.7317209119621474, + "0.8": 0.7362049043340335, + "0.9": 0.7515759927002796 + }, + "matched_mcc_abs": { + "0.6": 0.7183774821943227, + "0.7": 0.7297387567476799, + "0.8": 0.7355485459234283, + "0.9": 0.7507746583058363 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 73.65605666666666, + "l0_std": 0.3424813041653195, + "recon_r2_mean": 0.9227968597613038, + "recon_r2_std": 0.00016431300217292273, + "l1_per_token_mean": 484.92185180820735, + "l1_per_token_std": 1.1037586890827948, + "input_r2_mean": 0.9565933926880206, + "input_r2_std": 0.00038476439634421944, + "mcc_signed_mean": 0.6445646836170811, + "mcc_signed_std": 0.0052213028838355964, + "mcc_abs_mean": 0.6445906238491442, + "mcc_abs_std": 0.0052232296599457155, + "matched_mcc_signed_per_threshold": { + "0.6": 0.7258206500794895, + "0.7": 0.7325277379797815, + "0.8": 0.7390137621107226, + "0.9": 0.7525987017540549 + }, + "matched_mcc_signed_band": 0.737490212981012, + "matched_mcc_abs_per_threshold": { + "0.6": 0.7247536208062266, + "0.7": 0.7326785507735568, + "0.8": 0.7391659335239646, + "0.9": 0.7519617416095721 + }, + "matched_mcc_abs_band": 0.7371399616783301, + "act_iou_mean": 0.19559197006133414, + "act_iou_std": 0.002560105789443856, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/dict_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x4/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..18d4d37f00c72a5aba3022aa3eadb5e45e5fe47d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1551, + "0.6": 1033, + "0.7": 623, + "0.8": 334, + "0.9": 136 + }, + "n_dup_clusters_signed": { + "0.5": 591, + "0.6": 396, + "0.7": 233, + "0.8": 125, + "0.9": 50 + }, + "largest_dup_signed": { + "0.5": 26, + "0.6": 15, + "0.7": 12, + "0.8": 8, + "0.9": 8 + }, + "babel_mu1_signed": 0.5640567541122437, + "n_dup_abs": { + "0.5": 1562, + "0.6": 1045, + "0.7": 633, + "0.8": 342, + "0.9": 141 + }, + "n_dup_clusters_abs": { + "0.5": 594, + "0.6": 401, + "0.7": 238, + "0.8": 130, + "0.9": 53 + }, + "largest_dup_abs": { + "0.5": 26, + "0.6": 15, + "0.7": 12, + "0.8": 8, + "0.9": 8 + }, + "babel_mu1_abs": 0.5659437775611877 + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1534, + "0.6": 991, + "0.7": 601, + "0.8": 326, + "0.9": 139 + }, + "n_dup_clusters_signed": { + "0.5": 589, + "0.6": 371, + "0.7": 221, + "0.8": 120, + "0.9": 53 + }, + "largest_dup_signed": { + "0.5": 20, + "0.6": 17, + "0.7": 10, + "0.8": 9, + "0.9": 6 + }, + "babel_mu1_signed": 0.5614592432975769, + "n_dup_abs": { + "0.5": 1545, + "0.6": 1011, + "0.7": 615, + "0.8": 334, + "0.9": 139 + }, + "n_dup_clusters_abs": { + "0.5": 590, + "0.6": 380, + "0.7": 228, + "0.8": 124, + "0.9": 53 + }, + "largest_dup_abs": { + "0.5": 20, + "0.6": 17, + "0.7": 10, + "0.8": 9, + "0.9": 6 + }, + "babel_mu1_abs": 0.5635973811149597 + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1543, + "0.6": 986, + "0.7": 595, + "0.8": 317, + "0.9": 126 + }, + "n_dup_clusters_signed": { + "0.5": 595, + "0.6": 374, + "0.7": 218, + "0.8": 118, + "0.9": 47 + }, + "largest_dup_signed": { + "0.5": 27, + "0.6": 17, + "0.7": 13, + "0.8": 10, + "0.9": 6 + }, + "babel_mu1_signed": 0.5589005947113037, + "n_dup_abs": { + "0.5": 1550, + "0.6": 1000, + "0.7": 606, + "0.8": 326, + "0.9": 131 + }, + "n_dup_clusters_abs": { + "0.5": 597, + "0.6": 380, + "0.7": 223, + "0.8": 122, + "0.9": 49 + }, + "largest_dup_abs": { + "0.5": 27, + "0.6": 17, + "0.7": 13, + "0.8": 10, + "0.9": 6 + }, + "babel_mu1_abs": 0.561133086681366 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8712593913078308, + "dcs_abs": 0.873019278049469, + "frac_sign_flipped": 0.0146484375, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8646337389945984, + "0.6": 0.8683904409408569, + "0.7": 0.8698934316635132, + "0.8": 0.8715308904647827, + "0.9": 0.8716057538986206 + }, + "unique_frac_signed": { + "0.5": 0.68994140625, + "0.6": 0.79541015625, + "0.7": 0.8746744791666666, + "0.8": 0.9324544270833334, + "0.9": 0.9720052083333334 + }, + "dedup_dcs_signed": { + "0.5": 0.5965466177731287, + "0.6": 0.6907265763147734, + "0.7": 0.7608735842707877, + "0.8": 0.8126628371537663, + "0.9": 0.8472053324027609 + }, + "matched_frac_signed": { + "0.5": 0.6184895833333334, + "0.6": 0.7021484375, + "0.7": 0.7496744791666666, + "0.8": 0.7535807291666666, + "0.9": 0.6803385416666666 + }, + "matched_cos_abs": { + "0.5": 0.8662766814231873, + "0.6": 0.8699936866760254, + "0.7": 0.8714839816093445, + "0.8": 0.8729997277259827, + "0.9": 0.8736675381660461 + }, + "unique_frac_abs": { + "0.5": 0.68701171875, + "0.6": 0.79248046875, + "0.7": 0.8727213541666666, + "0.8": 0.9313151041666666, + "0.9": 0.9716796875 + }, + "dedup_dcs_abs": { + "0.5": 0.5951422318175901, + "0.6": 0.6894530046265572, + "0.7": 0.7605626805646655, + "0.8": 0.8130378323645951, + "0.9": 0.848925000464078 + }, + "matched_frac_abs": { + "0.5": 0.6168619791666666, + "0.6": 0.7018229166666666, + "0.7": 0.75, + "0.8": 0.7526041666666666, + "0.9": 0.681640625 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.873742401599884, + "dcs_abs": 0.8752098679542542, + "frac_sign_flipped": 0.015625, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8659340739250183, + "0.6": 0.8690500855445862, + "0.7": 0.8710234761238098, + "0.8": 0.872587263584137, + "0.9": 0.8741707801818848 + }, + "unique_frac_signed": { + "0.5": 0.689453125, + "0.6": 0.7967122395833334, + "0.7": 0.8751627604166666, + "0.8": 0.93359375, + "0.9": 0.97314453125 + }, + "dedup_dcs_signed": { + "0.5": 0.5970209533115849, + "0.6": 0.6923828399643147, + "0.7": 0.7622873097522339, + "0.8": 0.8146420156117529, + "0.9": 0.850694514112547 + }, + "matched_frac_signed": { + "0.5": 0.6197916666666666, + "0.6": 0.7001953125, + "0.7": 0.7483723958333334, + "0.8": 0.7496744791666666, + "0.9": 0.6904296875 + }, + "matched_cos_abs": { + "0.5": 0.8680004477500916, + "0.6": 0.870692253112793, + "0.7": 0.8725298643112183, + "0.8": 0.8738501667976379, + "0.9": 0.8755913972854614 + }, + "unique_frac_abs": { + "0.5": 0.6873372395833334, + "0.6": 0.7942708333333334, + "0.7": 0.8733723958333334, + "0.8": 0.9322916666666666, + "0.9": 0.9723307291666666 + }, + "dedup_dcs_abs": { + "0.5": 0.5966090317136453, + "0.6": 0.6915654614567757, + "0.7": 0.762043498029622, + "0.8": 0.8146832284207145, + "0.9": 0.8513644217746332 + }, + "matched_frac_abs": { + "0.5": 0.6184895833333334, + "0.6": 0.6988932291666666, + "0.7": 0.748046875, + "0.8": 0.7486979166666666, + "0.9": 0.689453125 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8670193552970886, + "dcs_abs": 0.8690447211265564, + "frac_sign_flipped": 0.013997395833333334, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8592876195907593, + "0.6": 0.8595407009124756, + "0.7": 0.8628034591674805, + "0.8": 0.865681529045105, + "0.9": 0.8668214678764343 + }, + "unique_frac_signed": { + "0.5": 0.69189453125, + "0.6": 0.7994791666666666, + "0.7": 0.8767903645833334, + "0.8": 0.93408203125, + "0.9": 0.97314453125 + }, + "dedup_dcs_signed": { + "0.5": 0.5945364047656767, + "0.6": 0.6871848832815886, + "0.7": 0.7564977595272164, + "0.8": 0.8086175610660575, + "0.9": 0.8435425710340496 + }, + "matched_frac_signed": { + "0.5": 0.6175130208333334, + "0.6": 0.69921875, + "0.7": 0.7431640625, + "0.8": 0.740234375, + "0.9": 0.6728515625 + }, + "matched_cos_abs": { + "0.5": 0.8613088726997375, + "0.6": 0.862220287322998, + "0.7": 0.8652018308639526, + "0.8": 0.8676667213439941, + "0.9": 0.8684054017066956 + }, + "unique_frac_abs": { + "0.5": 0.689453125, + "0.6": 0.79638671875, + "0.7": 0.8746744791666666, + "0.8": 0.9326171875, + "0.9": 0.97265625 + }, + "dedup_dcs_abs": { + "0.5": 0.5938320938730612, + "0.6": 0.6866607854608446, + "0.7": 0.7567699607849742, + "0.8": 0.809200897347182, + "0.9": 0.8446599415037781 + }, + "matched_frac_abs": { + "0.5": 0.6165364583333334, + "0.6": 0.7001953125, + "0.7": 0.7451171875, + "0.8": 0.7399088541666666, + "0.9": 0.6731770833333334 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.5614721973737081, + "babel_mu1_signed_std": 0.0025781041092325274, + "n_dup_signed_per_threshold": { + "0.5": 1542.6666666666667, + "0.6": 1003.3333333333334, + "0.7": 606.3333333333334, + "0.8": 325.6666666666667, + "0.9": 133.66666666666666 + }, + "n_dup_signed_band": 517.25, + "n_dup_signed_band_std": 13.012013679673105, + "n_dup_clusters_signed_per_threshold": { + "0.5": 591.6666666666666, + "0.6": 380.3333333333333, + "0.7": 224.0, + "0.8": 121.0, + "0.9": 50.0 + }, + "n_dup_clusters_signed_band": 193.83333333333331, + "n_dup_clusters_signed_band_std": 6.286559737514098, + "largest_dup_signed_per_threshold": { + "0.5": 24.333333333333332, + "0.6": 16.333333333333332, + "0.7": 11.666666666666666, + "0.8": 9.0, + "0.9": 6.666666666666667 + }, + "largest_dup_signed_band": 10.916666666666666, + "largest_dup_signed_band_std": 0.5204164998665332, + "dcs_signed_mean": 0.8706737160682678, + "dcs_signed_std": 0.003399573430896975, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5960346586167967, + "0.6": 0.690098099853559, + "0.7": 0.7598862178500795, + "0.8": 0.8119741379438589, + "0.9": 0.8471474725164526 + }, + "dedup_dcs_signed_band": 0.7772764820409875, + "dedup_dcs_signed_band_std": 0.003063487370290455, + "matched_cos_signed_per_threshold": { + "0.5": 0.8632851441701254, + "0.6": 0.8656604091326395, + "0.7": 0.8679067889849345, + "0.8": 0.8699332276980082, + "0.9": 0.8708660006523132 + }, + "matched_cos_signed_band": 0.8685916066169739, + "matched_cos_signed_band_std": 0.004279831918245436, + "unique_frac_signed_per_threshold": { + "0.5": 0.6904296875, + "0.6": 0.7972005208333334, + "0.7": 0.8755425347222222, + "0.8": 0.9333767361111112, + "0.9": 0.9727647569444445 + }, + "unique_frac_signed_band": 0.8947211371527779, + "unique_frac_signed_band_std": 0.0011205180963716663, + "matched_frac_signed_per_threshold": { + "0.5": 0.6185980902777778, + "0.6": 0.7005208333333334, + "0.7": 0.7470703125, + "0.8": 0.747829861111111, + "0.9": 0.6812065972222222 + }, + "matched_frac_signed_band": 0.7191569010416667, + "matched_frac_signed_band_std": 0.004595640593585747, + "babel_mu1_abs_mean": 0.5635580817858378, + "babel_mu1_abs_std": 0.0024055862098981574, + "n_dup_abs_per_threshold": { + "0.5": 1552.3333333333333, + "0.6": 1018.6666666666666, + "0.7": 618.0, + "0.8": 334.0, + "0.9": 137.0 + }, + "n_dup_abs_band": 526.9166666666666, + "n_dup_abs_band_std": 12.39287429668087, + "n_dup_clusters_abs_per_threshold": { + "0.5": 593.6666666666666, + "0.6": 387.0, + "0.7": 229.66666666666666, + "0.8": 125.33333333333333, + "0.9": 51.666666666666664 + }, + "n_dup_clusters_abs_band": 198.41666666666666, + "n_dup_clusters_abs_band_std": 6.286559737514098, + "largest_dup_abs_per_threshold": { + "0.5": 24.333333333333332, + "0.6": 16.333333333333332, + "0.7": 11.666666666666666, + "0.8": 9.0, + "0.9": 6.666666666666667 + }, + "largest_dup_abs_band": 10.916666666666666, + "largest_dup_abs_band_std": 0.5204164998665332, + "dcs_abs_mean": 0.8724246223767599, + "dcs_abs_std": 0.003125295246625796, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5951944524680989, + "0.6": 0.6892264171813925, + "0.7": 0.7597920464597538, + "0.8": 0.8123073193774971, + "0.9": 0.8483164545808298 + }, + "dedup_dcs_abs_band": 0.7774105593998684, + "dedup_dcs_abs_band_std": 0.0028410191373836094, + "matched_cos_abs_per_threshold": { + "0.5": 0.8651953339576721, + "0.6": 0.8676354090372721, + "0.7": 0.8697385589281718, + "0.8": 0.8715055386225382, + "0.9": 0.8725547790527344 + }, + "matched_cos_abs_band": 0.8703585714101791, + "matched_cos_abs_band_std": 0.003924989364999152, + "unique_frac_abs_per_threshold": { + "0.5": 0.6879340277777778, + "0.6": 0.7943793402777778, + "0.7": 0.8735894097222222, + "0.8": 0.9320746527777777, + "0.9": 0.9722222222222222 + }, + "unique_frac_abs_band": 0.89306640625, + "unique_frac_abs_band_std": 0.0010172526041666852, + "matched_frac_abs_per_threshold": { + "0.5": 0.6172960069444445, + "0.6": 0.7003038194444443, + "0.7": 0.7477213541666666, + "0.8": 0.7470703125, + "0.9": 0.6814236111111112 + }, + "matched_frac_abs_band": 0.7191297743055555, + "matched_frac_abs_band_std": 0.003925136543449168, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/config.json b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9b4a09489c3773c63b183c85e4f60cc2e9fb6270 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..611aeac5df14eb144edc0a378bcf7fb0fa1bff8e --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39dc3774ff84b39e7add87e4d3df89aef0e51f789bd21f2986ea4270726b6a46 +size 18890032 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3122e3fd708de8e9c4e6a746f7c239c3a2ac9338 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2942f0184a10c1c3014f541655f933769fed233dcdad0e8c4d3b57667f84e3d1 +size 25920 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..592622f4c1e6512e1cf184ec091a514836299d02 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9c7892e424fdf73f7e05109df0020fefb51674e77774b3b76c119aa7341fe93 +size 13607 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..d2813f51e8e5de7bd5c99af4351551c062a32d25 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 10.475146484375, + "l0_per_token_0.001": 10.475146484375, + "l0_per_token_0.01": 10.475146484375, + "l0_per_token_0.05": 10.475146484375, + "l0_per_token_0.1": 10.475146484375, + "l0_per_token_0.5": 10.475146484375, + "l0_per_token_1.0": 10.475035791015625, + "l1_per_token": 230.85932195708597, + "r2_score": 0.8580868174619836, + "l2_loss": 3.3282095238444085 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1af8f5badba46926133acc3f1bffda6a52edfd0f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a8fa759ded2d025d4c1f89f90e18bd4f2badbc244ab787fc67726d3e91e2130 +size 13518 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8faf090cac1df732c618bb206a3c64ebdd50fad9 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2817f9d244a8ef66137350b51d44cf38a901becc70a1b7be0b973ab305309f2 +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..78326748f1f11add3dfe13e2948e2d3489ebc305 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b9072665afd9d74e0282ff43aeccfe4eeec61c0a4c66af5ddc494dfc42fa753 +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/config.json b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..77d01289e11fb5754255e1d8a654b6e6d25adf36 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b23f11e3e0e3ba5cf0fdfd435364926468d9fa50 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ece4953e78b78e51d125b37145db1c5e835fbd23d6f8d94bba572f5e83137832 +size 18890032 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..18a7dc5ef5811f24e8c5d68404f292159faf6646 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edf684f868810caea592330d4bea53da536cb33766e8280948e4c70cc4fb7d17 +size 25920 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7afa501ca76526c8723707637a93a17b4538f33b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a2d70a4f20e5ced577405d74255eed00c3431ad2f8a73c0d04a1be1a09db3bd +size 13607 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b050e743a5802175e58157d9411c7bd5a1c254f4 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 10.56886103515625, + "l0_per_token_0.001": 10.56886103515625, + "l0_per_token_0.01": 10.56886103515625, + "l0_per_token_0.05": 10.56886103515625, + "l0_per_token_0.1": 10.56886103515625, + "l0_per_token_0.5": 10.56886103515625, + "l0_per_token_1.0": 10.568568505859375, + "l1_per_token": 231.99529593014606, + "r2_score": 0.8585413067072317, + "l2_loss": 3.3175506448913255 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e7e898c95f8d4fec609b1e9a17e1a04af5a81ea --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4adaa6040fc4f6a935670521b8728d84e8539e020a7d61817f23a56eb2ae3aea +size 13518 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d97e90a7e1694e4d7c7149a9c47966e98ddedc1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f5e11b979c83cdf2c27b67dc437bdf8340959c353f35db7165c61e88eb44caa +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e65612db80893d2e00c55d438655fa98a264881d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:643030770d17313fc0b3ad4d41f5228a9375a61cadd9caf5d8eadb04a686bc7b +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/config.json b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..44bc89e8d578c36cb7f3fbf06dbbf9d15cd3712d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/model.safetensors b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..293fa4c8be0843168440f254039206d7ccda0428 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d1ca72fb8c7f2e1805bbbba139e43111dbe3d26b224a4e2b1eabec8a0b8847c +size 18890032 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..eeb8f7431d23ffc5b688b0e36d3feb6b169c4bf1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f873561c1547b6c9afa2cc468bd5f9c3b395408d3205884b815743f1dc4288ff +size 25920 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..88ca72d3e9691e7ac967f9322819df59f7255b88 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbd62d8d78ae1246259d53b1706d42325f325062c000ed27f83583697028af07 +size 13607 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..a7b8f7bba0abb20aa23810d24adf6565f04dac33 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 10.695765283203125, + "l0_per_token_0.001": 10.695765283203125, + "l0_per_token_0.01": 10.695765283203125, + "l0_per_token_0.05": 10.695765283203125, + "l0_per_token_0.1": 10.695765283203125, + "l0_per_token_0.5": 10.695765283203125, + "l0_per_token_1.0": 10.695730712890626, + "l1_per_token": 232.68004341909048, + "r2_score": 0.8590042224397443, + "l2_loss": 3.306694144303205 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bb0e46378d1f084df520e3d34287d6f5d44bb19 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fa66e16ad4dc36377bd93fbed804fc09a7e6f9a261b55c6b51f6100a7b49fba +size 13518 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d633317ec36a49440b3f625ed9535dd23470bdc2 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4e9ea6fa43c329a29282f294d8f0077f4502204130a35867170498912c569a4 +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..94e9466cfd65e13a0572c0f95673975c395d543b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b18aff3ac7dff83bb2c82320a782a7e8857b5d7fbb3bcdceddb86e2305f22b2 +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/code_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x4/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..ba1efb88b6e7c7c96030648a3d292a2a27ae16b4 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 73.26211666666667, + "recon_r2": 0.9226123499770492, + "l1_per_token": 483.66945199742497 + }, + "seed1": { + "l0": 73.88305833333334, + "recon_r2": 0.9228508280066877, + "l1_per_token": 485.7527570608418 + }, + "seed2": { + "l0": 73.822995, + "recon_r2": 0.9229274013001743, + "l1_per_token": 485.3433463663552 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6457484252590913, + "mcc_abs": 0.6457676334652961, + "act_iou": 0.19626021556076378, + "act_iou_n_pairs": 3072, + "input_r2": 0.9566984295000585, + "matched_mcc_signed": { + "0.6": 0.7282515952041155, + "0.7": 0.7317495131780405, + "0.8": 0.7411449981760442, + "0.9": 0.7545275621589508 + }, + "matched_mcc_abs": { + "0.6": 0.7265693911147596, + "0.7": 0.7333936392047536, + "0.8": 0.7418744218329142, + "0.9": 0.7535220217329011 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6490924876488843, + "mcc_abs": 0.6491249224096499, + "act_iou": 0.19775168524089884, + "act_iou_n_pairs": 3072, + "input_r2": 0.9569147313292561, + "matched_mcc_signed": { + "0.6": 0.7281977294534634, + "0.7": 0.7341127887991568, + "0.8": 0.7396913838220901, + "0.9": 0.7516925504029341 + }, + "matched_mcc_abs": { + "0.6": 0.7293139891095975, + "0.7": 0.7349032563682367, + "0.8": 0.7400748328155513, + "0.9": 0.7515885447899787 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6388531379432676, + "mcc_abs": 0.6388793156724867, + "act_iou": 0.19276400938233987, + "act_iou_n_pairs": 3072, + "input_r2": 0.9561670172347472, + "matched_mcc_signed": { + "0.6": 0.7210126255808897, + "0.7": 0.7317209119621474, + "0.8": 0.7362049043340335, + "0.9": 0.7515759927002796 + }, + "matched_mcc_abs": { + "0.6": 0.7183774821943227, + "0.7": 0.7297387567476799, + "0.8": 0.7355485459234283, + "0.9": 0.7507746583058363 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 73.65605666666666, + "l0_std": 0.3424813041653195, + "recon_r2_mean": 0.9227968597613038, + "recon_r2_std": 0.00016431300217292273, + "l1_per_token_mean": 484.92185180820735, + "l1_per_token_std": 1.1037586890827948, + "input_r2_mean": 0.9565933926880206, + "input_r2_std": 0.00038476439634421944, + "mcc_signed_mean": 0.6445646836170811, + "mcc_signed_std": 0.0052213028838355964, + "mcc_abs_mean": 0.6445906238491442, + "mcc_abs_std": 0.0052232296599457155, + "matched_mcc_signed_per_threshold": { + "0.6": 0.7258206500794895, + "0.7": 0.7325277379797815, + "0.8": 0.7390137621107226, + "0.9": 0.7525987017540549 + }, + "matched_mcc_signed_band": 0.737490212981012, + "matched_mcc_abs_per_threshold": { + "0.6": 0.7247536208062266, + "0.7": 0.7326785507735568, + "0.8": 0.7391659335239646, + "0.9": 0.7519617416095721 + }, + "matched_mcc_abs_band": 0.7371399616783301, + "act_iou_mean": 0.19559197006133414, + "act_iou_std": 0.002560105789443856, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/dict_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x4/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..18d4d37f00c72a5aba3022aa3eadb5e45e5fe47d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1551, + "0.6": 1033, + "0.7": 623, + "0.8": 334, + "0.9": 136 + }, + "n_dup_clusters_signed": { + "0.5": 591, + "0.6": 396, + "0.7": 233, + "0.8": 125, + "0.9": 50 + }, + "largest_dup_signed": { + "0.5": 26, + "0.6": 15, + "0.7": 12, + "0.8": 8, + "0.9": 8 + }, + "babel_mu1_signed": 0.5640567541122437, + "n_dup_abs": { + "0.5": 1562, + "0.6": 1045, + "0.7": 633, + "0.8": 342, + "0.9": 141 + }, + "n_dup_clusters_abs": { + "0.5": 594, + "0.6": 401, + "0.7": 238, + "0.8": 130, + "0.9": 53 + }, + "largest_dup_abs": { + "0.5": 26, + "0.6": 15, + "0.7": 12, + "0.8": 8, + "0.9": 8 + }, + "babel_mu1_abs": 0.5659437775611877 + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1534, + "0.6": 991, + "0.7": 601, + "0.8": 326, + "0.9": 139 + }, + "n_dup_clusters_signed": { + "0.5": 589, + "0.6": 371, + "0.7": 221, + "0.8": 120, + "0.9": 53 + }, + "largest_dup_signed": { + "0.5": 20, + "0.6": 17, + "0.7": 10, + "0.8": 9, + "0.9": 6 + }, + "babel_mu1_signed": 0.5614592432975769, + "n_dup_abs": { + "0.5": 1545, + "0.6": 1011, + "0.7": 615, + "0.8": 334, + "0.9": 139 + }, + "n_dup_clusters_abs": { + "0.5": 590, + "0.6": 380, + "0.7": 228, + "0.8": 124, + "0.9": 53 + }, + "largest_dup_abs": { + "0.5": 20, + "0.6": 17, + "0.7": 10, + "0.8": 9, + "0.9": 6 + }, + "babel_mu1_abs": 0.5635973811149597 + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1543, + "0.6": 986, + "0.7": 595, + "0.8": 317, + "0.9": 126 + }, + "n_dup_clusters_signed": { + "0.5": 595, + "0.6": 374, + "0.7": 218, + "0.8": 118, + "0.9": 47 + }, + "largest_dup_signed": { + "0.5": 27, + "0.6": 17, + "0.7": 13, + "0.8": 10, + "0.9": 6 + }, + "babel_mu1_signed": 0.5589005947113037, + "n_dup_abs": { + "0.5": 1550, + "0.6": 1000, + "0.7": 606, + "0.8": 326, + "0.9": 131 + }, + "n_dup_clusters_abs": { + "0.5": 597, + "0.6": 380, + "0.7": 223, + "0.8": 122, + "0.9": 49 + }, + "largest_dup_abs": { + "0.5": 27, + "0.6": 17, + "0.7": 13, + "0.8": 10, + "0.9": 6 + }, + "babel_mu1_abs": 0.561133086681366 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8712593913078308, + "dcs_abs": 0.873019278049469, + "frac_sign_flipped": 0.0146484375, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8646337389945984, + "0.6": 0.8683904409408569, + "0.7": 0.8698934316635132, + "0.8": 0.8715308904647827, + "0.9": 0.8716057538986206 + }, + "unique_frac_signed": { + "0.5": 0.68994140625, + "0.6": 0.79541015625, + "0.7": 0.8746744791666666, + "0.8": 0.9324544270833334, + "0.9": 0.9720052083333334 + }, + "dedup_dcs_signed": { + "0.5": 0.5965466177731287, + "0.6": 0.6907265763147734, + "0.7": 0.7608735842707877, + "0.8": 0.8126628371537663, + "0.9": 0.8472053324027609 + }, + "matched_frac_signed": { + "0.5": 0.6184895833333334, + "0.6": 0.7021484375, + "0.7": 0.7496744791666666, + "0.8": 0.7535807291666666, + "0.9": 0.6803385416666666 + }, + "matched_cos_abs": { + "0.5": 0.8662766814231873, + "0.6": 0.8699936866760254, + "0.7": 0.8714839816093445, + "0.8": 0.8729997277259827, + "0.9": 0.8736675381660461 + }, + "unique_frac_abs": { + "0.5": 0.68701171875, + "0.6": 0.79248046875, + "0.7": 0.8727213541666666, + "0.8": 0.9313151041666666, + "0.9": 0.9716796875 + }, + "dedup_dcs_abs": { + "0.5": 0.5951422318175901, + "0.6": 0.6894530046265572, + "0.7": 0.7605626805646655, + "0.8": 0.8130378323645951, + "0.9": 0.848925000464078 + }, + "matched_frac_abs": { + "0.5": 0.6168619791666666, + "0.6": 0.7018229166666666, + "0.7": 0.75, + "0.8": 0.7526041666666666, + "0.9": 0.681640625 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.873742401599884, + "dcs_abs": 0.8752098679542542, + "frac_sign_flipped": 0.015625, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8659340739250183, + "0.6": 0.8690500855445862, + "0.7": 0.8710234761238098, + "0.8": 0.872587263584137, + "0.9": 0.8741707801818848 + }, + "unique_frac_signed": { + "0.5": 0.689453125, + "0.6": 0.7967122395833334, + "0.7": 0.8751627604166666, + "0.8": 0.93359375, + "0.9": 0.97314453125 + }, + "dedup_dcs_signed": { + "0.5": 0.5970209533115849, + "0.6": 0.6923828399643147, + "0.7": 0.7622873097522339, + "0.8": 0.8146420156117529, + "0.9": 0.850694514112547 + }, + "matched_frac_signed": { + "0.5": 0.6197916666666666, + "0.6": 0.7001953125, + "0.7": 0.7483723958333334, + "0.8": 0.7496744791666666, + "0.9": 0.6904296875 + }, + "matched_cos_abs": { + "0.5": 0.8680004477500916, + "0.6": 0.870692253112793, + "0.7": 0.8725298643112183, + "0.8": 0.8738501667976379, + "0.9": 0.8755913972854614 + }, + "unique_frac_abs": { + "0.5": 0.6873372395833334, + "0.6": 0.7942708333333334, + "0.7": 0.8733723958333334, + "0.8": 0.9322916666666666, + "0.9": 0.9723307291666666 + }, + "dedup_dcs_abs": { + "0.5": 0.5966090317136453, + "0.6": 0.6915654614567757, + "0.7": 0.762043498029622, + "0.8": 0.8146832284207145, + "0.9": 0.8513644217746332 + }, + "matched_frac_abs": { + "0.5": 0.6184895833333334, + "0.6": 0.6988932291666666, + "0.7": 0.748046875, + "0.8": 0.7486979166666666, + "0.9": 0.689453125 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8670193552970886, + "dcs_abs": 0.8690447211265564, + "frac_sign_flipped": 0.013997395833333334, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8592876195907593, + "0.6": 0.8595407009124756, + "0.7": 0.8628034591674805, + "0.8": 0.865681529045105, + "0.9": 0.8668214678764343 + }, + "unique_frac_signed": { + "0.5": 0.69189453125, + "0.6": 0.7994791666666666, + "0.7": 0.8767903645833334, + "0.8": 0.93408203125, + "0.9": 0.97314453125 + }, + "dedup_dcs_signed": { + "0.5": 0.5945364047656767, + "0.6": 0.6871848832815886, + "0.7": 0.7564977595272164, + "0.8": 0.8086175610660575, + "0.9": 0.8435425710340496 + }, + "matched_frac_signed": { + "0.5": 0.6175130208333334, + "0.6": 0.69921875, + "0.7": 0.7431640625, + "0.8": 0.740234375, + "0.9": 0.6728515625 + }, + "matched_cos_abs": { + "0.5": 0.8613088726997375, + "0.6": 0.862220287322998, + "0.7": 0.8652018308639526, + "0.8": 0.8676667213439941, + "0.9": 0.8684054017066956 + }, + "unique_frac_abs": { + "0.5": 0.689453125, + "0.6": 0.79638671875, + "0.7": 0.8746744791666666, + "0.8": 0.9326171875, + "0.9": 0.97265625 + }, + "dedup_dcs_abs": { + "0.5": 0.5938320938730612, + "0.6": 0.6866607854608446, + "0.7": 0.7567699607849742, + "0.8": 0.809200897347182, + "0.9": 0.8446599415037781 + }, + "matched_frac_abs": { + "0.5": 0.6165364583333334, + "0.6": 0.7001953125, + "0.7": 0.7451171875, + "0.8": 0.7399088541666666, + "0.9": 0.6731770833333334 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.5614721973737081, + "babel_mu1_signed_std": 0.0025781041092325274, + "n_dup_signed_per_threshold": { + "0.5": 1542.6666666666667, + "0.6": 1003.3333333333334, + "0.7": 606.3333333333334, + "0.8": 325.6666666666667, + "0.9": 133.66666666666666 + }, + "n_dup_signed_band": 517.25, + "n_dup_signed_band_std": 13.012013679673105, + "n_dup_clusters_signed_per_threshold": { + "0.5": 591.6666666666666, + "0.6": 380.3333333333333, + "0.7": 224.0, + "0.8": 121.0, + "0.9": 50.0 + }, + "n_dup_clusters_signed_band": 193.83333333333331, + "n_dup_clusters_signed_band_std": 6.286559737514098, + "largest_dup_signed_per_threshold": { + "0.5": 24.333333333333332, + "0.6": 16.333333333333332, + "0.7": 11.666666666666666, + "0.8": 9.0, + "0.9": 6.666666666666667 + }, + "largest_dup_signed_band": 10.916666666666666, + "largest_dup_signed_band_std": 0.5204164998665332, + "dcs_signed_mean": 0.8706737160682678, + "dcs_signed_std": 0.003399573430896975, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5960346586167967, + "0.6": 0.690098099853559, + "0.7": 0.7598862178500795, + "0.8": 0.8119741379438589, + "0.9": 0.8471474725164526 + }, + "dedup_dcs_signed_band": 0.7772764820409875, + "dedup_dcs_signed_band_std": 0.003063487370290455, + "matched_cos_signed_per_threshold": { + "0.5": 0.8632851441701254, + "0.6": 0.8656604091326395, + "0.7": 0.8679067889849345, + "0.8": 0.8699332276980082, + "0.9": 0.8708660006523132 + }, + "matched_cos_signed_band": 0.8685916066169739, + "matched_cos_signed_band_std": 0.004279831918245436, + "unique_frac_signed_per_threshold": { + "0.5": 0.6904296875, + "0.6": 0.7972005208333334, + "0.7": 0.8755425347222222, + "0.8": 0.9333767361111112, + "0.9": 0.9727647569444445 + }, + "unique_frac_signed_band": 0.8947211371527779, + "unique_frac_signed_band_std": 0.0011205180963716663, + "matched_frac_signed_per_threshold": { + "0.5": 0.6185980902777778, + "0.6": 0.7005208333333334, + "0.7": 0.7470703125, + "0.8": 0.747829861111111, + "0.9": 0.6812065972222222 + }, + "matched_frac_signed_band": 0.7191569010416667, + "matched_frac_signed_band_std": 0.004595640593585747, + "babel_mu1_abs_mean": 0.5635580817858378, + "babel_mu1_abs_std": 0.0024055862098981574, + "n_dup_abs_per_threshold": { + "0.5": 1552.3333333333333, + "0.6": 1018.6666666666666, + "0.7": 618.0, + "0.8": 334.0, + "0.9": 137.0 + }, + "n_dup_abs_band": 526.9166666666666, + "n_dup_abs_band_std": 12.39287429668087, + "n_dup_clusters_abs_per_threshold": { + "0.5": 593.6666666666666, + "0.6": 387.0, + "0.7": 229.66666666666666, + "0.8": 125.33333333333333, + "0.9": 51.666666666666664 + }, + "n_dup_clusters_abs_band": 198.41666666666666, + "n_dup_clusters_abs_band_std": 6.286559737514098, + "largest_dup_abs_per_threshold": { + "0.5": 24.333333333333332, + "0.6": 16.333333333333332, + "0.7": 11.666666666666666, + "0.8": 9.0, + "0.9": 6.666666666666667 + }, + "largest_dup_abs_band": 10.916666666666666, + "largest_dup_abs_band_std": 0.5204164998665332, + "dcs_abs_mean": 0.8724246223767599, + "dcs_abs_std": 0.003125295246625796, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5951944524680989, + "0.6": 0.6892264171813925, + "0.7": 0.7597920464597538, + "0.8": 0.8123073193774971, + "0.9": 0.8483164545808298 + }, + "dedup_dcs_abs_band": 0.7774105593998684, + "dedup_dcs_abs_band_std": 0.0028410191373836094, + "matched_cos_abs_per_threshold": { + "0.5": 0.8651953339576721, + "0.6": 0.8676354090372721, + "0.7": 0.8697385589281718, + "0.8": 0.8715055386225382, + "0.9": 0.8725547790527344 + }, + "matched_cos_abs_band": 0.8703585714101791, + "matched_cos_abs_band_std": 0.003924989364999152, + "unique_frac_abs_per_threshold": { + "0.5": 0.6879340277777778, + "0.6": 0.7943793402777778, + "0.7": 0.8735894097222222, + "0.8": 0.9320746527777777, + "0.9": 0.9722222222222222 + }, + "unique_frac_abs_band": 0.89306640625, + "unique_frac_abs_band_std": 0.0010172526041666852, + "matched_frac_abs_per_threshold": { + "0.5": 0.6172960069444445, + "0.6": 0.7003038194444443, + "0.7": 0.7477213541666666, + "0.8": 0.7470703125, + "0.9": 0.6814236111111112 + }, + "matched_frac_abs_band": 0.7191297743055555, + "matched_frac_abs_band_std": 0.003925136543449168, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/config.json b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9b4a09489c3773c63b183c85e4f60cc2e9fb6270 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/model.safetensors b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..611aeac5df14eb144edc0a378bcf7fb0fa1bff8e --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39dc3774ff84b39e7add87e4d3df89aef0e51f789bd21f2986ea4270726b6a46 +size 18890032 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3122e3fd708de8e9c4e6a746f7c239c3a2ac9338 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2942f0184a10c1c3014f541655f933769fed233dcdad0e8c4d3b57667f84e3d1 +size 25920 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..592622f4c1e6512e1cf184ec091a514836299d02 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9c7892e424fdf73f7e05109df0020fefb51674e77774b3b76c119aa7341fe93 +size 13607 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..d2813f51e8e5de7bd5c99af4351551c062a32d25 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 10.475146484375, + "l0_per_token_0.001": 10.475146484375, + "l0_per_token_0.01": 10.475146484375, + "l0_per_token_0.05": 10.475146484375, + "l0_per_token_0.1": 10.475146484375, + "l0_per_token_0.5": 10.475146484375, + "l0_per_token_1.0": 10.475035791015625, + "l1_per_token": 230.85932195708597, + "r2_score": 0.8580868174619836, + "l2_loss": 3.3282095238444085 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1af8f5badba46926133acc3f1bffda6a52edfd0f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a8fa759ded2d025d4c1f89f90e18bd4f2badbc244ab787fc67726d3e91e2130 +size 13518 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8faf090cac1df732c618bb206a3c64ebdd50fad9 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2817f9d244a8ef66137350b51d44cf38a901becc70a1b7be0b973ab305309f2 +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..78326748f1f11add3dfe13e2948e2d3489ebc305 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b9072665afd9d74e0282ff43aeccfe4eeec61c0a4c66af5ddc494dfc42fa753 +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/config.json b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..77d01289e11fb5754255e1d8a654b6e6d25adf36 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/model.safetensors b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b23f11e3e0e3ba5cf0fdfd435364926468d9fa50 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ece4953e78b78e51d125b37145db1c5e835fbd23d6f8d94bba572f5e83137832 +size 18890032 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..18a7dc5ef5811f24e8c5d68404f292159faf6646 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edf684f868810caea592330d4bea53da536cb33766e8280948e4c70cc4fb7d17 +size 25920 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7afa501ca76526c8723707637a93a17b4538f33b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a2d70a4f20e5ced577405d74255eed00c3431ad2f8a73c0d04a1be1a09db3bd +size 13607 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b050e743a5802175e58157d9411c7bd5a1c254f4 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 10.56886103515625, + "l0_per_token_0.001": 10.56886103515625, + "l0_per_token_0.01": 10.56886103515625, + "l0_per_token_0.05": 10.56886103515625, + "l0_per_token_0.1": 10.56886103515625, + "l0_per_token_0.5": 10.56886103515625, + "l0_per_token_1.0": 10.568568505859375, + "l1_per_token": 231.99529593014606, + "r2_score": 0.8585413067072317, + "l2_loss": 3.3175506448913255 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e7e898c95f8d4fec609b1e9a17e1a04af5a81ea --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4adaa6040fc4f6a935670521b8728d84e8539e020a7d61817f23a56eb2ae3aea +size 13518 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d97e90a7e1694e4d7c7149a9c47966e98ddedc1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f5e11b979c83cdf2c27b67dc437bdf8340959c353f35db7165c61e88eb44caa +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e65612db80893d2e00c55d438655fa98a264881d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:643030770d17313fc0b3ad4d41f5228a9375a61cadd9caf5d8eadb04a686bc7b +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/config.json b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..44bc89e8d578c36cb7f3fbf06dbbf9d15cd3712d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/model.safetensors b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..293fa4c8be0843168440f254039206d7ccda0428 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d1ca72fb8c7f2e1805bbbba139e43111dbe3d26b224a4e2b1eabec8a0b8847c +size 18890032 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..eeb8f7431d23ffc5b688b0e36d3feb6b169c4bf1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f873561c1547b6c9afa2cc468bd5f9c3b395408d3205884b815743f1dc4288ff +size 25920 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..88ca72d3e9691e7ac967f9322819df59f7255b88 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbd62d8d78ae1246259d53b1706d42325f325062c000ed27f83583697028af07 +size 13607 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..a7b8f7bba0abb20aa23810d24adf6565f04dac33 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 10.695765283203125, + "l0_per_token_0.001": 10.695765283203125, + "l0_per_token_0.01": 10.695765283203125, + "l0_per_token_0.05": 10.695765283203125, + "l0_per_token_0.1": 10.695765283203125, + "l0_per_token_0.5": 10.695765283203125, + "l0_per_token_1.0": 10.695730712890626, + "l1_per_token": 232.68004341909048, + "r2_score": 0.8590042224397443, + "l2_loss": 3.306694144303205 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bb0e46378d1f084df520e3d34287d6f5d44bb19 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fa66e16ad4dc36377bd93fbed804fc09a7e6f9a261b55c6b51f6100a7b49fba +size 13518 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d633317ec36a49440b3f625ed9535dd23470bdc2 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4e9ea6fa43c329a29282f294d8f0077f4502204130a35867170498912c569a4 +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..94e9466cfd65e13a0572c0f95673975c395d543b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b18aff3ac7dff83bb2c82320a782a7e8857b5d7fbb3bcdceddb86e2305f22b2 +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/code_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x4/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..ba1efb88b6e7c7c96030648a3d292a2a27ae16b4 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 73.26211666666667, + "recon_r2": 0.9226123499770492, + "l1_per_token": 483.66945199742497 + }, + "seed1": { + "l0": 73.88305833333334, + "recon_r2": 0.9228508280066877, + "l1_per_token": 485.7527570608418 + }, + "seed2": { + "l0": 73.822995, + "recon_r2": 0.9229274013001743, + "l1_per_token": 485.3433463663552 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6457484252590913, + "mcc_abs": 0.6457676334652961, + "act_iou": 0.19626021556076378, + "act_iou_n_pairs": 3072, + "input_r2": 0.9566984295000585, + "matched_mcc_signed": { + "0.6": 0.7282515952041155, + "0.7": 0.7317495131780405, + "0.8": 0.7411449981760442, + "0.9": 0.7545275621589508 + }, + "matched_mcc_abs": { + "0.6": 0.7265693911147596, + "0.7": 0.7333936392047536, + "0.8": 0.7418744218329142, + "0.9": 0.7535220217329011 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6490924876488843, + "mcc_abs": 0.6491249224096499, + "act_iou": 0.19775168524089884, + "act_iou_n_pairs": 3072, + "input_r2": 0.9569147313292561, + "matched_mcc_signed": { + "0.6": 0.7281977294534634, + "0.7": 0.7341127887991568, + "0.8": 0.7396913838220901, + "0.9": 0.7516925504029341 + }, + "matched_mcc_abs": { + "0.6": 0.7293139891095975, + "0.7": 0.7349032563682367, + "0.8": 0.7400748328155513, + "0.9": 0.7515885447899787 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6388531379432676, + "mcc_abs": 0.6388793156724867, + "act_iou": 0.19276400938233987, + "act_iou_n_pairs": 3072, + "input_r2": 0.9561670172347472, + "matched_mcc_signed": { + "0.6": 0.7210126255808897, + "0.7": 0.7317209119621474, + "0.8": 0.7362049043340335, + "0.9": 0.7515759927002796 + }, + "matched_mcc_abs": { + "0.6": 0.7183774821943227, + "0.7": 0.7297387567476799, + "0.8": 0.7355485459234283, + "0.9": 0.7507746583058363 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 73.65605666666666, + "l0_std": 0.3424813041653195, + "recon_r2_mean": 0.9227968597613038, + "recon_r2_std": 0.00016431300217292273, + "l1_per_token_mean": 484.92185180820735, + "l1_per_token_std": 1.1037586890827948, + "input_r2_mean": 0.9565933926880206, + "input_r2_std": 0.00038476439634421944, + "mcc_signed_mean": 0.6445646836170811, + "mcc_signed_std": 0.0052213028838355964, + "mcc_abs_mean": 0.6445906238491442, + "mcc_abs_std": 0.0052232296599457155, + "matched_mcc_signed_per_threshold": { + "0.6": 0.7258206500794895, + "0.7": 0.7325277379797815, + "0.8": 0.7390137621107226, + "0.9": 0.7525987017540549 + }, + "matched_mcc_signed_band": 0.737490212981012, + "matched_mcc_abs_per_threshold": { + "0.6": 0.7247536208062266, + "0.7": 0.7326785507735568, + "0.8": 0.7391659335239646, + "0.9": 0.7519617416095721 + }, + "matched_mcc_abs_band": 0.7371399616783301, + "act_iou_mean": 0.19559197006133414, + "act_iou_std": 0.002560105789443856, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/dict_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x4/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..18d4d37f00c72a5aba3022aa3eadb5e45e5fe47d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1551, + "0.6": 1033, + "0.7": 623, + "0.8": 334, + "0.9": 136 + }, + "n_dup_clusters_signed": { + "0.5": 591, + "0.6": 396, + "0.7": 233, + "0.8": 125, + "0.9": 50 + }, + "largest_dup_signed": { + "0.5": 26, + "0.6": 15, + "0.7": 12, + "0.8": 8, + "0.9": 8 + }, + "babel_mu1_signed": 0.5640567541122437, + "n_dup_abs": { + "0.5": 1562, + "0.6": 1045, + "0.7": 633, + "0.8": 342, + "0.9": 141 + }, + "n_dup_clusters_abs": { + "0.5": 594, + "0.6": 401, + "0.7": 238, + "0.8": 130, + "0.9": 53 + }, + "largest_dup_abs": { + "0.5": 26, + "0.6": 15, + "0.7": 12, + "0.8": 8, + "0.9": 8 + }, + "babel_mu1_abs": 0.5659437775611877 + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1534, + "0.6": 991, + "0.7": 601, + "0.8": 326, + "0.9": 139 + }, + "n_dup_clusters_signed": { + "0.5": 589, + "0.6": 371, + "0.7": 221, + "0.8": 120, + "0.9": 53 + }, + "largest_dup_signed": { + "0.5": 20, + "0.6": 17, + "0.7": 10, + "0.8": 9, + "0.9": 6 + }, + "babel_mu1_signed": 0.5614592432975769, + "n_dup_abs": { + "0.5": 1545, + "0.6": 1011, + "0.7": 615, + "0.8": 334, + "0.9": 139 + }, + "n_dup_clusters_abs": { + "0.5": 590, + "0.6": 380, + "0.7": 228, + "0.8": 124, + "0.9": 53 + }, + "largest_dup_abs": { + "0.5": 20, + "0.6": 17, + "0.7": 10, + "0.8": 9, + "0.9": 6 + }, + "babel_mu1_abs": 0.5635973811149597 + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1543, + "0.6": 986, + "0.7": 595, + "0.8": 317, + "0.9": 126 + }, + "n_dup_clusters_signed": { + "0.5": 595, + "0.6": 374, + "0.7": 218, + "0.8": 118, + "0.9": 47 + }, + "largest_dup_signed": { + "0.5": 27, + "0.6": 17, + "0.7": 13, + "0.8": 10, + "0.9": 6 + }, + "babel_mu1_signed": 0.5589005947113037, + "n_dup_abs": { + "0.5": 1550, + "0.6": 1000, + "0.7": 606, + "0.8": 326, + "0.9": 131 + }, + "n_dup_clusters_abs": { + "0.5": 597, + "0.6": 380, + "0.7": 223, + "0.8": 122, + "0.9": 49 + }, + "largest_dup_abs": { + "0.5": 27, + "0.6": 17, + "0.7": 13, + "0.8": 10, + "0.9": 6 + }, + "babel_mu1_abs": 0.561133086681366 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8712593913078308, + "dcs_abs": 0.873019278049469, + "frac_sign_flipped": 0.0146484375, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8646337389945984, + "0.6": 0.8683904409408569, + "0.7": 0.8698934316635132, + "0.8": 0.8715308904647827, + "0.9": 0.8716057538986206 + }, + "unique_frac_signed": { + "0.5": 0.68994140625, + "0.6": 0.79541015625, + "0.7": 0.8746744791666666, + "0.8": 0.9324544270833334, + "0.9": 0.9720052083333334 + }, + "dedup_dcs_signed": { + "0.5": 0.5965466177731287, + "0.6": 0.6907265763147734, + "0.7": 0.7608735842707877, + "0.8": 0.8126628371537663, + "0.9": 0.8472053324027609 + }, + "matched_frac_signed": { + "0.5": 0.6184895833333334, + "0.6": 0.7021484375, + "0.7": 0.7496744791666666, + "0.8": 0.7535807291666666, + "0.9": 0.6803385416666666 + }, + "matched_cos_abs": { + "0.5": 0.8662766814231873, + "0.6": 0.8699936866760254, + "0.7": 0.8714839816093445, + "0.8": 0.8729997277259827, + "0.9": 0.8736675381660461 + }, + "unique_frac_abs": { + "0.5": 0.68701171875, + "0.6": 0.79248046875, + "0.7": 0.8727213541666666, + "0.8": 0.9313151041666666, + "0.9": 0.9716796875 + }, + "dedup_dcs_abs": { + "0.5": 0.5951422318175901, + "0.6": 0.6894530046265572, + "0.7": 0.7605626805646655, + "0.8": 0.8130378323645951, + "0.9": 0.848925000464078 + }, + "matched_frac_abs": { + "0.5": 0.6168619791666666, + "0.6": 0.7018229166666666, + "0.7": 0.75, + "0.8": 0.7526041666666666, + "0.9": 0.681640625 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.873742401599884, + "dcs_abs": 0.8752098679542542, + "frac_sign_flipped": 0.015625, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8659340739250183, + "0.6": 0.8690500855445862, + "0.7": 0.8710234761238098, + "0.8": 0.872587263584137, + "0.9": 0.8741707801818848 + }, + "unique_frac_signed": { + "0.5": 0.689453125, + "0.6": 0.7967122395833334, + "0.7": 0.8751627604166666, + "0.8": 0.93359375, + "0.9": 0.97314453125 + }, + "dedup_dcs_signed": { + "0.5": 0.5970209533115849, + "0.6": 0.6923828399643147, + "0.7": 0.7622873097522339, + "0.8": 0.8146420156117529, + "0.9": 0.850694514112547 + }, + "matched_frac_signed": { + "0.5": 0.6197916666666666, + "0.6": 0.7001953125, + "0.7": 0.7483723958333334, + "0.8": 0.7496744791666666, + "0.9": 0.6904296875 + }, + "matched_cos_abs": { + "0.5": 0.8680004477500916, + "0.6": 0.870692253112793, + "0.7": 0.8725298643112183, + "0.8": 0.8738501667976379, + "0.9": 0.8755913972854614 + }, + "unique_frac_abs": { + "0.5": 0.6873372395833334, + "0.6": 0.7942708333333334, + "0.7": 0.8733723958333334, + "0.8": 0.9322916666666666, + "0.9": 0.9723307291666666 + }, + "dedup_dcs_abs": { + "0.5": 0.5966090317136453, + "0.6": 0.6915654614567757, + "0.7": 0.762043498029622, + "0.8": 0.8146832284207145, + "0.9": 0.8513644217746332 + }, + "matched_frac_abs": { + "0.5": 0.6184895833333334, + "0.6": 0.6988932291666666, + "0.7": 0.748046875, + "0.8": 0.7486979166666666, + "0.9": 0.689453125 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8670193552970886, + "dcs_abs": 0.8690447211265564, + "frac_sign_flipped": 0.013997395833333334, + "alive_source": "stats", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8592876195907593, + "0.6": 0.8595407009124756, + "0.7": 0.8628034591674805, + "0.8": 0.865681529045105, + "0.9": 0.8668214678764343 + }, + "unique_frac_signed": { + "0.5": 0.69189453125, + "0.6": 0.7994791666666666, + "0.7": 0.8767903645833334, + "0.8": 0.93408203125, + "0.9": 0.97314453125 + }, + "dedup_dcs_signed": { + "0.5": 0.5945364047656767, + "0.6": 0.6871848832815886, + "0.7": 0.7564977595272164, + "0.8": 0.8086175610660575, + "0.9": 0.8435425710340496 + }, + "matched_frac_signed": { + "0.5": 0.6175130208333334, + "0.6": 0.69921875, + "0.7": 0.7431640625, + "0.8": 0.740234375, + "0.9": 0.6728515625 + }, + "matched_cos_abs": { + "0.5": 0.8613088726997375, + "0.6": 0.862220287322998, + "0.7": 0.8652018308639526, + "0.8": 0.8676667213439941, + "0.9": 0.8684054017066956 + }, + "unique_frac_abs": { + "0.5": 0.689453125, + "0.6": 0.79638671875, + "0.7": 0.8746744791666666, + "0.8": 0.9326171875, + "0.9": 0.97265625 + }, + "dedup_dcs_abs": { + "0.5": 0.5938320938730612, + "0.6": 0.6866607854608446, + "0.7": 0.7567699607849742, + "0.8": 0.809200897347182, + "0.9": 0.8446599415037781 + }, + "matched_frac_abs": { + "0.5": 0.6165364583333334, + "0.6": 0.7001953125, + "0.7": 0.7451171875, + "0.8": 0.7399088541666666, + "0.9": 0.6731770833333334 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.5614721973737081, + "babel_mu1_signed_std": 0.0025781041092325274, + "n_dup_signed_per_threshold": { + "0.5": 1542.6666666666667, + "0.6": 1003.3333333333334, + "0.7": 606.3333333333334, + "0.8": 325.6666666666667, + "0.9": 133.66666666666666 + }, + "n_dup_signed_band": 517.25, + "n_dup_signed_band_std": 13.012013679673105, + "n_dup_clusters_signed_per_threshold": { + "0.5": 591.6666666666666, + "0.6": 380.3333333333333, + "0.7": 224.0, + "0.8": 121.0, + "0.9": 50.0 + }, + "n_dup_clusters_signed_band": 193.83333333333331, + "n_dup_clusters_signed_band_std": 6.286559737514098, + "largest_dup_signed_per_threshold": { + "0.5": 24.333333333333332, + "0.6": 16.333333333333332, + "0.7": 11.666666666666666, + "0.8": 9.0, + "0.9": 6.666666666666667 + }, + "largest_dup_signed_band": 10.916666666666666, + "largest_dup_signed_band_std": 0.5204164998665332, + "dcs_signed_mean": 0.8706737160682678, + "dcs_signed_std": 0.003399573430896975, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5960346586167967, + "0.6": 0.690098099853559, + "0.7": 0.7598862178500795, + "0.8": 0.8119741379438589, + "0.9": 0.8471474725164526 + }, + "dedup_dcs_signed_band": 0.7772764820409875, + "dedup_dcs_signed_band_std": 0.003063487370290455, + "matched_cos_signed_per_threshold": { + "0.5": 0.8632851441701254, + "0.6": 0.8656604091326395, + "0.7": 0.8679067889849345, + "0.8": 0.8699332276980082, + "0.9": 0.8708660006523132 + }, + "matched_cos_signed_band": 0.8685916066169739, + "matched_cos_signed_band_std": 0.004279831918245436, + "unique_frac_signed_per_threshold": { + "0.5": 0.6904296875, + "0.6": 0.7972005208333334, + "0.7": 0.8755425347222222, + "0.8": 0.9333767361111112, + "0.9": 0.9727647569444445 + }, + "unique_frac_signed_band": 0.8947211371527779, + "unique_frac_signed_band_std": 0.0011205180963716663, + "matched_frac_signed_per_threshold": { + "0.5": 0.6185980902777778, + "0.6": 0.7005208333333334, + "0.7": 0.7470703125, + "0.8": 0.747829861111111, + "0.9": 0.6812065972222222 + }, + "matched_frac_signed_band": 0.7191569010416667, + "matched_frac_signed_band_std": 0.004595640593585747, + "babel_mu1_abs_mean": 0.5635580817858378, + "babel_mu1_abs_std": 0.0024055862098981574, + "n_dup_abs_per_threshold": { + "0.5": 1552.3333333333333, + "0.6": 1018.6666666666666, + "0.7": 618.0, + "0.8": 334.0, + "0.9": 137.0 + }, + "n_dup_abs_band": 526.9166666666666, + "n_dup_abs_band_std": 12.39287429668087, + "n_dup_clusters_abs_per_threshold": { + "0.5": 593.6666666666666, + "0.6": 387.0, + "0.7": 229.66666666666666, + "0.8": 125.33333333333333, + "0.9": 51.666666666666664 + }, + "n_dup_clusters_abs_band": 198.41666666666666, + "n_dup_clusters_abs_band_std": 6.286559737514098, + "largest_dup_abs_per_threshold": { + "0.5": 24.333333333333332, + "0.6": 16.333333333333332, + "0.7": 11.666666666666666, + "0.8": 9.0, + "0.9": 6.666666666666667 + }, + "largest_dup_abs_band": 10.916666666666666, + "largest_dup_abs_band_std": 0.5204164998665332, + "dcs_abs_mean": 0.8724246223767599, + "dcs_abs_std": 0.003125295246625796, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5951944524680989, + "0.6": 0.6892264171813925, + "0.7": 0.7597920464597538, + "0.8": 0.8123073193774971, + "0.9": 0.8483164545808298 + }, + "dedup_dcs_abs_band": 0.7774105593998684, + "dedup_dcs_abs_band_std": 0.0028410191373836094, + "matched_cos_abs_per_threshold": { + "0.5": 0.8651953339576721, + "0.6": 0.8676354090372721, + "0.7": 0.8697385589281718, + "0.8": 0.8715055386225382, + "0.9": 0.8725547790527344 + }, + "matched_cos_abs_band": 0.8703585714101791, + "matched_cos_abs_band_std": 0.003924989364999152, + "unique_frac_abs_per_threshold": { + "0.5": 0.6879340277777778, + "0.6": 0.7943793402777778, + "0.7": 0.8735894097222222, + "0.8": 0.9320746527777777, + "0.9": 0.9722222222222222 + }, + "unique_frac_abs_band": 0.89306640625, + "unique_frac_abs_band_std": 0.0010172526041666852, + "matched_frac_abs_per_threshold": { + "0.5": 0.6172960069444445, + "0.6": 0.7003038194444443, + "0.7": 0.7477213541666666, + "0.8": 0.7470703125, + "0.9": 0.6814236111111112 + }, + "matched_frac_abs_band": 0.7191297743055555, + "matched_frac_abs_band_std": 0.003925136543449168, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/config.json b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9b4a09489c3773c63b183c85e4f60cc2e9fb6270 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/model.safetensors b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..611aeac5df14eb144edc0a378bcf7fb0fa1bff8e --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39dc3774ff84b39e7add87e4d3df89aef0e51f789bd21f2986ea4270726b6a46 +size 18890032 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3122e3fd708de8e9c4e6a746f7c239c3a2ac9338 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2942f0184a10c1c3014f541655f933769fed233dcdad0e8c4d3b57667f84e3d1 +size 25920 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..592622f4c1e6512e1cf184ec091a514836299d02 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9c7892e424fdf73f7e05109df0020fefb51674e77774b3b76c119aa7341fe93 +size 13607 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..d2813f51e8e5de7bd5c99af4351551c062a32d25 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 10.475146484375, + "l0_per_token_0.001": 10.475146484375, + "l0_per_token_0.01": 10.475146484375, + "l0_per_token_0.05": 10.475146484375, + "l0_per_token_0.1": 10.475146484375, + "l0_per_token_0.5": 10.475146484375, + "l0_per_token_1.0": 10.475035791015625, + "l1_per_token": 230.85932195708597, + "r2_score": 0.8580868174619836, + "l2_loss": 3.3282095238444085 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1af8f5badba46926133acc3f1bffda6a52edfd0f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a8fa759ded2d025d4c1f89f90e18bd4f2badbc244ab787fc67726d3e91e2130 +size 13518 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8faf090cac1df732c618bb206a3c64ebdd50fad9 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2817f9d244a8ef66137350b51d44cf38a901becc70a1b7be0b973ab305309f2 +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..78326748f1f11add3dfe13e2948e2d3489ebc305 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b9072665afd9d74e0282ff43aeccfe4eeec61c0a4c66af5ddc494dfc42fa753 +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/config.json b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..77d01289e11fb5754255e1d8a654b6e6d25adf36 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/model.safetensors b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b23f11e3e0e3ba5cf0fdfd435364926468d9fa50 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ece4953e78b78e51d125b37145db1c5e835fbd23d6f8d94bba572f5e83137832 +size 18890032 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..18a7dc5ef5811f24e8c5d68404f292159faf6646 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edf684f868810caea592330d4bea53da536cb33766e8280948e4c70cc4fb7d17 +size 25920 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7afa501ca76526c8723707637a93a17b4538f33b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a2d70a4f20e5ced577405d74255eed00c3431ad2f8a73c0d04a1be1a09db3bd +size 13607 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b050e743a5802175e58157d9411c7bd5a1c254f4 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 10.56886103515625, + "l0_per_token_0.001": 10.56886103515625, + "l0_per_token_0.01": 10.56886103515625, + "l0_per_token_0.05": 10.56886103515625, + "l0_per_token_0.1": 10.56886103515625, + "l0_per_token_0.5": 10.56886103515625, + "l0_per_token_1.0": 10.568568505859375, + "l1_per_token": 231.99529593014606, + "r2_score": 0.8585413067072317, + "l2_loss": 3.3175506448913255 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e7e898c95f8d4fec609b1e9a17e1a04af5a81ea --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4adaa6040fc4f6a935670521b8728d84e8539e020a7d61817f23a56eb2ae3aea +size 13518 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d97e90a7e1694e4d7c7149a9c47966e98ddedc1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f5e11b979c83cdf2c27b67dc437bdf8340959c353f35db7165c61e88eb44caa +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e65612db80893d2e00c55d438655fa98a264881d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:643030770d17313fc0b3ad4d41f5228a9375a61cadd9caf5d8eadb04a686bc7b +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/config.json b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..44bc89e8d578c36cb7f3fbf06dbbf9d15cd3712d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/model.safetensors b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..293fa4c8be0843168440f254039206d7ccda0428 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d1ca72fb8c7f2e1805bbbba139e43111dbe3d26b224a4e2b1eabec8a0b8847c +size 18890032 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..eeb8f7431d23ffc5b688b0e36d3feb6b169c4bf1 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f873561c1547b6c9afa2cc468bd5f9c3b395408d3205884b815743f1dc4288ff +size 25920 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..88ca72d3e9691e7ac967f9322819df59f7255b88 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbd62d8d78ae1246259d53b1706d42325f325062c000ed27f83583697028af07 +size 13607 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..a7b8f7bba0abb20aa23810d24adf6565f04dac33 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 10.695765283203125, + "l0_per_token_0.001": 10.695765283203125, + "l0_per_token_0.01": 10.695765283203125, + "l0_per_token_0.05": 10.695765283203125, + "l0_per_token_0.1": 10.695765283203125, + "l0_per_token_0.5": 10.695765283203125, + "l0_per_token_1.0": 10.695730712890626, + "l1_per_token": 232.68004341909048, + "r2_score": 0.8590042224397443, + "l2_loss": 3.306694144303205 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bb0e46378d1f084df520e3d34287d6f5d44bb19 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fa66e16ad4dc36377bd93fbed804fc09a7e6f9a261b55c6b51f6100a7b49fba +size 13518 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d633317ec36a49440b3f625ed9535dd23470bdc2 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4e9ea6fa43c329a29282f294d8f0077f4502204130a35867170498912c569a4 +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..94e9466cfd65e13a0572c0f95673975c395d543b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b18aff3ac7dff83bb2c82320a782a7e8857b5d7fbb3bcdceddb86e2305f22b2 +size 13523 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/code_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x8/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..1a6a07ebaf52d0a6eb85722fafe6b26b36959f32 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 133.342685, + "recon_r2": 0.9579364822230559, + "l1_per_token": 622.5739052568949 + }, + "seed1": { + "l0": 133.74438666666666, + "recon_r2": 0.9580450849409071, + "l1_per_token": 624.8499954625632 + }, + "seed2": { + "l0": 132.899125, + "recon_r2": 0.9579131916194232, + "l1_per_token": 622.6106457408799 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.5577321366719193, + "mcc_abs": 0.557746485916359, + "act_iou": 0.12213332597887794, + "act_iou_n_pairs": 6144, + "input_r2": 0.9688372132773408, + "matched_mcc_signed": { + "0.6": 0.642655610066397, + "0.7": 0.6531531421587754, + "0.8": 0.6600240988435896, + "0.9": 0.6771347244925351 + }, + "matched_mcc_abs": { + "0.6": 0.6411153437110967, + "0.7": 0.651264146359182, + "0.8": 0.6593221182755177, + "0.9": 0.6763283143523651 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5555783110863869, + "mcc_abs": 0.5555893378922883, + "act_iou": 0.12190028999310154, + "act_iou_n_pairs": 6144, + "input_r2": 0.9688121912263958, + "matched_mcc_signed": { + "0.6": 0.6417433238566577, + "0.7": 0.6537775509211723, + "0.8": 0.6626166110304657, + "0.9": 0.677604842646902 + }, + "matched_mcc_abs": { + "0.6": 0.6406682577340987, + "0.7": 0.6541223723378987, + "0.8": 0.6618975300687223, + "0.9": 0.6771529459612937 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5573900481677448, + "mcc_abs": 0.5574042106974981, + "act_iou": 0.1224582018667922, + "act_iou_n_pairs": 6144, + "input_r2": 0.968922839675886, + "matched_mcc_signed": { + "0.6": 0.6428978422155754, + "0.7": 0.6520874050185033, + "0.8": 0.6611660515592243, + "0.9": 0.6789412539771904 + }, + "matched_mcc_abs": { + "0.6": 0.6407262362148244, + "0.7": 0.6505745275770458, + "0.8": 0.6607224377840826, + "0.9": 0.6784005985357081 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 133.32873222222221, + "l0_std": 0.4228035374613265, + "recon_r2_mean": 0.9579649195944621, + "recon_r2_std": 7.039513571534565e-05, + "l1_per_token_mean": 623.3448488201128, + "l1_per_token_std": 1.3036246688564024, + "input_r2_mean": 0.968857414726541, + "input_r2_std": 5.802450995898652e-05, + "mcc_signed_mean": 0.5569001653086837, + "mcc_signed_std": 0.0011574670945322814, + "mcc_abs_mean": 0.5569133448353818, + "mcc_abs_std": 0.0011593247475929613, + "matched_mcc_signed_per_threshold": { + "0.6": 0.6424322587128768, + "0.7": 0.6530060326994837, + "0.8": 0.6612689204777599, + "0.9": 0.6778936070388758 + }, + "matched_mcc_signed_band": 0.658650204732249, + "matched_mcc_abs_per_threshold": { + "0.6": 0.6408366125533399, + "0.7": 0.6519870154247088, + "0.8": 0.6606473620427742, + "0.9": 0.677293952949789 + }, + "matched_mcc_abs_band": 0.657691235742653, + "act_iou_mean": 0.12216393927959056, + "act_iou_std": 0.00028021294641155596, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/dict_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x8/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..84972e78d27e9cd92625f17df698d6853a895702 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3640, + "0.6": 2442, + "0.7": 1557, + "0.8": 821, + "0.9": 339 + }, + "n_dup_clusters_signed": { + "0.5": 1333, + "0.6": 894, + "0.7": 573, + "0.8": 300, + "0.9": 131 + }, + "largest_dup_signed": { + "0.5": 37, + "0.6": 17, + "0.7": 16, + "0.8": 11, + "0.9": 9 + }, + "babel_mu1_signed": 0.6008203625679016, + "n_dup_abs": { + "0.5": 3655, + "0.6": 2476, + "0.7": 1573, + "0.8": 836, + "0.9": 339 + }, + "n_dup_clusters_abs": { + "0.5": 1335, + "0.6": 908, + "0.7": 580, + "0.8": 306, + "0.9": 131 + }, + "largest_dup_abs": { + "0.5": 37, + "0.6": 17, + "0.7": 16, + "0.8": 11, + "0.9": 9 + }, + "babel_mu1_abs": 0.6029527187347412 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3642, + "0.6": 2486, + "0.7": 1536, + "0.8": 868, + "0.9": 317 + }, + "n_dup_clusters_signed": { + "0.5": 1317, + "0.6": 908, + "0.7": 557, + "0.8": 324, + "0.9": 125 + }, + "largest_dup_signed": { + "0.5": 34, + "0.6": 28, + "0.7": 13, + "0.8": 11, + "0.9": 8 + }, + "babel_mu1_signed": 0.6018854975700378, + "n_dup_abs": { + "0.5": 3663, + "0.6": 2513, + "0.7": 1557, + "0.8": 882, + "0.9": 320 + }, + "n_dup_clusters_abs": { + "0.5": 1324, + "0.6": 919, + "0.7": 566, + "0.8": 331, + "0.9": 126 + }, + "largest_dup_abs": { + "0.5": 34, + "0.6": 30, + "0.7": 14, + "0.8": 11, + "0.9": 8 + }, + "babel_mu1_abs": 0.6040351390838623 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3597, + "0.6": 2418, + "0.7": 1556, + "0.8": 848, + "0.9": 322 + }, + "n_dup_clusters_signed": { + "0.5": 1295, + "0.6": 869, + "0.7": 559, + "0.8": 303, + "0.9": 115 + }, + "largest_dup_signed": { + "0.5": 37, + "0.6": 25, + "0.7": 17, + "0.8": 11, + "0.9": 9 + }, + "babel_mu1_signed": 0.5993590950965881, + "n_dup_abs": { + "0.5": 3615, + "0.6": 2442, + "0.7": 1583, + "0.8": 860, + "0.9": 327 + }, + "n_dup_clusters_abs": { + "0.5": 1300, + "0.6": 879, + "0.7": 573, + "0.8": 310, + "0.9": 118 + }, + "largest_dup_abs": { + "0.5": 45, + "0.6": 22, + "0.7": 17, + "0.8": 11, + "0.9": 9 + }, + "babel_mu1_abs": 0.6011876463890076 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8542901873588562, + "dcs_abs": 0.8564583659172058, + "frac_sign_flipped": 0.017415364583333332, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8426259160041809, + "0.6": 0.8466602563858032, + "0.7": 0.8478602766990662, + "0.8": 0.8523461818695068, + "0.9": 0.8540911078453064 + }, + "unique_frac_signed": { + "0.5": 0.623046875, + "0.6": 0.74560546875, + "0.7": 0.8402506510416666, + "0.8": 0.913330078125, + "0.9": 0.9674479166666666 + }, + "dedup_dcs_signed": { + "0.5": 0.5249954437604174, + "0.6": 0.631274517334532, + "0.7": 0.712415149488758, + "0.8": 0.7784734048764221, + "0.9": 0.8262886629284669 + }, + "matched_frac_signed": { + "0.5": 0.546875, + "0.6": 0.6399739583333334, + "0.7": 0.6829427083333334, + "0.8": 0.6884765625, + "0.9": 0.6181640625 + }, + "matched_cos_abs": { + "0.5": 0.846027672290802, + "0.6": 0.848763644695282, + "0.7": 0.8503178358078003, + "0.8": 0.8545229434967041, + "0.9": 0.8560903072357178 + }, + "unique_frac_abs": { + "0.5": 0.620849609375, + "0.6": 0.74267578125, + "0.7": 0.8385416666666666, + "0.8": 0.9120279947916666, + "0.9": 0.96728515625 + }, + "dedup_dcs_abs": { + "0.5": 0.5252559498621849, + "0.6": 0.630356202920666, + "0.7": 0.7130269352346659, + "0.8": 0.7793488466607716, + "0.9": 0.8280834465986118 + }, + "matched_frac_abs": { + "0.5": 0.54833984375, + "0.6": 0.6394856770833334, + "0.7": 0.68603515625, + "0.8": 0.6884765625, + "0.9": 0.6181640625 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8556812405586243, + "dcs_abs": 0.857814610004425, + "frac_sign_flipped": 0.0166015625, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8444643020629883, + "0.6": 0.847666323184967, + "0.7": 0.8498727083206177, + "0.8": 0.8532578349113464, + "0.9": 0.8539199233055115 + }, + "unique_frac_signed": { + "0.5": 0.6249186197916666, + "0.6": 0.7479654947916666, + "0.7": 0.8387858072916666, + "0.8": 0.9132486979166666, + "0.9": 0.9662272135416666 + }, + "dedup_dcs_signed": { + "0.5": 0.5277214661085357, + "0.6": 0.6340251608392766, + "0.7": 0.7128611657438644, + "0.8": 0.7792366067199812, + "0.9": 0.825080668083198 + }, + "matched_frac_signed": { + "0.5": 0.552734375, + "0.6": 0.6476236979166666, + "0.7": 0.6868489583333334, + "0.8": 0.6847330729166666, + "0.9": 0.6181640625 + }, + "matched_cos_abs": { + "0.5": 0.8471776843070984, + "0.6": 0.8505780100822449, + "0.7": 0.8527635335922241, + "0.8": 0.8553610444068909, + "0.9": 0.8560706377029419 + }, + "unique_frac_abs": { + "0.5": 0.622802734375, + "0.6": 0.7451985677083334, + "0.7": 0.8369954427083334, + "0.8": 0.912109375, + "0.9": 0.966064453125 + }, + "dedup_dcs_abs": { + "0.5": 0.5276245782879414, + "0.6": 0.6338495148374932, + "0.7": 0.7137591913245463, + "0.8": 0.7801828276133165, + "0.9": 0.8270194124488626 + }, + "matched_frac_abs": { + "0.5": 0.5524088541666666, + "0.6": 0.6487630208333334, + "0.7": 0.6884765625, + "0.8": 0.6861979166666666, + "0.9": 0.61865234375 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8569879531860352, + "dcs_abs": 0.8593170046806335, + "frac_sign_flipped": 0.017740885416666668, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8467940092086792, + "0.6": 0.850313127040863, + "0.7": 0.8523238897323608, + "0.8": 0.8527654409408569, + "0.9": 0.8570263385772705 + }, + "unique_frac_signed": { + "0.5": 0.6234537760416666, + "0.6": 0.7455240885416666, + "0.7": 0.8391927083333334, + "0.8": 0.911376953125, + "0.9": 0.967529296875 + }, + "dedup_dcs_signed": { + "0.5": 0.5279369225706129, + "0.6": 0.6339289190121538, + "0.7": 0.7152639934017013, + "0.8": 0.7771907692949753, + "0.9": 0.8291980907670222 + }, + "matched_frac_signed": { + "0.5": 0.5520833333333334, + "0.6": 0.6463216145833334, + "0.7": 0.6930338541666666, + "0.8": 0.6863606770833334, + "0.9": 0.6181640625 + }, + "matched_cos_abs": { + "0.5": 0.8501198291778564, + "0.6": 0.8531087040901184, + "0.7": 0.8549855947494507, + "0.8": 0.8550905585289001, + "0.9": 0.8593066930770874 + }, + "unique_frac_abs": { + "0.5": 0.6212565104166666, + "0.6": 0.7430826822916666, + "0.7": 0.837158203125, + "0.8": 0.910400390625, + "0.9": 0.9672037760416666 + }, + "dedup_dcs_abs": { + "0.5": 0.5281424785110478, + "0.6": 0.6339303041216529, + "0.7": 0.7157582041982096, + "0.8": 0.7784747785044601, + "0.9": 0.8311246783220364 + }, + "matched_frac_abs": { + "0.5": 0.5525716145833334, + "0.6": 0.64794921875, + "0.7": 0.6954752604166666, + "0.8": 0.68798828125, + "0.9": 0.6189778645833334 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.6006883184115092, + "babel_mu1_signed_std": 0.0012683667091544298, + "n_dup_signed_per_threshold": { + "0.5": 3626.3333333333335, + "0.6": 2448.6666666666665, + "0.7": 1549.6666666666667, + "0.8": 845.6666666666666, + "0.9": 326.0 + }, + "n_dup_signed_band": 1292.5, + "n_dup_signed_band_std": 8.227241335952167, + "n_dup_clusters_signed_per_threshold": { + "0.5": 1315.0, + "0.6": 890.3333333333334, + "0.7": 563.0, + "0.8": 309.0, + "0.9": 123.66666666666667 + }, + "n_dup_clusters_signed_band": 471.50000000000006, + "n_dup_clusters_signed_band_std": 8.888194417315589, + "largest_dup_signed_per_threshold": { + "0.5": 36.0, + "0.6": 23.333333333333332, + "0.7": 15.333333333333334, + "0.8": 11.0, + "0.9": 8.666666666666666 + }, + "largest_dup_signed_band": 14.583333333333332, + "largest_dup_signed_band_std": 1.1814539065631522, + "dcs_signed_mean": 0.8556531270345052, + "dcs_signed_std": 0.001349102624803774, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5268846108131887, + "0.6": 0.6330761990619874, + "0.7": 0.7135134362114414, + "0.8": 0.7783002602971262, + "0.9": 0.8268558072595624 + }, + "dedup_dcs_signed_band": 0.7379364257075294, + "dedup_dcs_signed_band_std": 0.0008989495748107668, + "matched_cos_signed_per_threshold": { + "0.5": 0.8446280757586161, + "0.6": 0.8482132355372111, + "0.7": 0.8500189582506815, + "0.8": 0.8527898192405701, + "0.9": 0.8550124565760294 + }, + "matched_cos_signed_band": 0.8515086174011229, + "matched_cos_signed_band_std": 0.0014619767956287413, + "unique_frac_signed_per_threshold": { + "0.5": 0.623806423611111, + "0.6": 0.746365017361111, + "0.7": 0.8394097222222222, + "0.8": 0.9126519097222222, + "0.9": 0.967068142361111 + }, + "unique_frac_signed_band": 0.8663736979166666, + "unique_frac_signed_band_std": 0.0004084240702287405, + "matched_frac_signed_per_threshold": { + "0.5": 0.5505642361111112, + "0.6": 0.6446397569444445, + "0.7": 0.6876085069444445, + "0.8": 0.6865234375, + "0.9": 0.6181640625 + }, + "matched_frac_signed_band": 0.6592339409722223, + "matched_frac_signed_band_std": 0.0017928289541946734, + "babel_mu1_abs_mean": 0.6027251680692037, + "babel_mu1_abs_std": 0.0014373197768251494, + "n_dup_abs_per_threshold": { + "0.5": 3644.3333333333335, + "0.6": 2477.0, + "0.7": 1571.0, + "0.8": 859.3333333333334, + "0.9": 328.6666666666667 + }, + "n_dup_abs_band": 1309.0, + "n_dup_abs_band_std": 7.937253933193772, + "n_dup_clusters_abs_per_threshold": { + "0.5": 1319.6666666666667, + "0.6": 902.0, + "0.7": 573.0, + "0.8": 315.6666666666667, + "0.9": 125.0 + }, + "n_dup_clusters_abs_band": 478.9166666666667, + "n_dup_clusters_abs_band_std": 8.009109397013711, + "largest_dup_abs_per_threshold": { + "0.5": 38.666666666666664, + "0.6": 23.0, + "0.7": 15.666666666666666, + "0.8": 11.0, + "0.9": 8.666666666666666 + }, + "largest_dup_abs_band": 14.583333333333332, + "largest_dup_abs_band_std": 1.2583057392117918, + "dcs_abs_mean": 0.8578633268674215, + "dcs_abs_std": 0.0014299419199736995, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5270076688870581, + "0.6": 0.6327120072932707, + "0.7": 0.7141814435858073, + "0.8": 0.779335484259516, + "0.9": 0.8287425124565035 + }, + "dedup_dcs_abs_band": 0.7387428618987744, + "dedup_dcs_abs_band_std": 0.0010596366557693272, + "matched_cos_abs_per_threshold": { + "0.5": 0.8477750619252523, + "0.6": 0.8508167862892151, + "0.7": 0.8526889880498251, + "0.8": 0.8549915154774984, + "0.9": 0.8571558793385824 + }, + "matched_cos_abs_band": 0.8539132922887803, + "matched_cos_abs_band_std": 0.001610907561081223, + "unique_frac_abs_per_threshold": { + "0.5": 0.6216362847222222, + "0.6": 0.74365234375, + "0.7": 0.8375651041666666, + "0.8": 0.9115125868055555, + "0.9": 0.9668511284722222 + }, + "unique_frac_abs_band": 0.864895290798611, + "unique_frac_abs_band_std": 0.00037642928294214797, + "matched_frac_abs_per_threshold": { + "0.5": 0.5511067708333334, + "0.6": 0.6453993055555556, + "0.7": 0.6899956597222222, + "0.8": 0.6875542534722222, + "0.9": 0.6185980902777778 + }, + "matched_frac_abs_band": 0.6603868272569444, + "matched_frac_abs_band_std": 0.002281671362234943, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/config.json b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b3fd244772bb48fe3d86fad8c03b0af36f99e8b2 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8f76f7fc734b7117511af49de2af3ea854b59441 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54b3e94850029a0a46bfff3e50f3fb6d0387c84ea48732bff3a701dc95c4620a +size 37776696 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b08cd466c0fc441f7a73337549f9fdf1d2d93a20 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f39464ddf437e3449e6d46420fc94b35e4c260d0927e88b014ed85d938e772be +size 50496 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..dff0dc4aea686df670cf6619179697a0d23c4701 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51db8f375b183d1aeb73855da60f6c816c492894103aa3d9e931eb6cbc4f8ac4 +size 25895 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..42e57c584b8debcc03eefad9e1450b434ece69fa --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 14.916806689453125, + "l0_per_token_0.001": 14.916806689453125, + "l0_per_token_0.01": 14.916806689453125, + "l0_per_token_0.05": 14.916806689453125, + "l0_per_token_0.1": 14.916806689453125, + "l0_per_token_0.5": 14.916806689453125, + "l0_per_token_1.0": 14.9143650390625, + "l1_per_token": 265.01689360429793, + "r2_score": 0.8894365591273903, + "l2_loss": 2.5929817816794567 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6209f14801e05486cb3e7dde7c49fc8a752420e3 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce5e0e9cb2c9ead1e49b096475699c24c94b65637a7f162c7fe2a5372c32cb7a +size 25806 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dddcd4fbfbfd6978385e42dc9fe20a9856dd79cd --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fd2ee0fe4307c0d94179cabb6707cf0b120c8ef6b79c8374bb6ed8abec9eacd +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c577a6fb3f080d663222adebd6219daffcf9d10 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5d72fc10deb95387849e64a3f949bc6544540436a895698433c0ca91d98376e +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/config.json b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b57334bdfb7c241cea507faaf65ed199052d3e8f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c71cc3dcc80526a12a63ee2692d7edfe53c6fcd3 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd56808a0cfdc11d9582a46d55cb652a59bc102a1ddb7ec32c63ebcd6ae7ac8 +size 37776696 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2edb01a73dcc85e66f2471c71a1c28f94bbded0c --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ea7cdc1373f6fbe47cdfad71c99448b86d6594e8bfa37716e7177159b7c147d +size 50496 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fcbe1c7974baaf9d6165fb32b731eb7e7b1b03b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3663faa374844d8072a644c6a69dadf2a03e014053cfd7e6c15911eb00163768 +size 25895 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e0e3b01c5070f294f3793715ad6c9a66bd588088 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 15.187090625, + "l0_per_token_0.001": 15.187090625, + "l0_per_token_0.01": 15.187090625, + "l0_per_token_0.05": 15.187090625, + "l0_per_token_0.1": 15.187090625, + "l0_per_token_0.5": 15.187090625, + "l0_per_token_1.0": 15.185013623046874, + "l1_per_token": 267.8888524205817, + "r2_score": 0.8899659762104499, + "l2_loss": 2.5805656625676696 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ad83d80f397eded9c4ef94e9962762bd9870bcb --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f737f4ff74d8038c4d46c5ce56701809047a496a9b844f6fa26bf6eb669ce389 +size 25806 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e116b343c8e74a0774f8e811a1f37b525831296 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d44b766dff5cac3fee2e5306f20dabc603d157432f89cccb1956c664e9ebebfb +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..950f6b3bd8e8e353e0bb4cb4c5616ffadbab2925 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45ecb96c70650646097e184a9d63ae587736459149d7809db40ccb800083429e +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/config.json b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..abd1311d294b93d49cdd3c8a13e2ddb9120dc92d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/model.safetensors b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e64fc257fef848dc974f9f771edcb57d012935c0 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0debc5f746cea79ad38d56a650ae8440f211bdd45f62a8a9444311f3059cbf0f +size 37776696 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7cb685c6fe1be659e22aa49942e695b94e6fe277 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9980ecc1196c6833ff6867ad58ec89b65cac6c1cdc78ddb91a6bf1bc4e872413 +size 50496 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c2497504142f40c430757a8010b9279b9f66432 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0449412d8f787f76bdc76a3c8b1cd4c104038723bfccc7f900079dc6ba1278e2 +size 25895 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..15ca1f9f09b990e6c6da9aa1e2ef1c8beca6f3f3 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 14.739921923828126, + "l0_per_token_0.001": 14.739921923828126, + "l0_per_token_0.01": 14.739921923828126, + "l0_per_token_0.05": 14.739921923828126, + "l0_per_token_0.1": 14.739921923828126, + "l0_per_token_0.5": 14.739921923828126, + "l0_per_token_1.0": 14.738592138671875, + "l1_per_token": 264.7434869000165, + "r2_score": 0.8891176971020024, + "l2_loss": 2.60045987223249 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2740a3900247d39e8fbf2127c06089cbd95c151 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d5f4d1fd929f4536767dd8c1367dd1ee9d52d6995ba5711134b6967c27ee00f +size 25806 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f77b46bf07755d01509d39beb5d00e151d3d47f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:913d5dd80ddb868913dc945914371061307245c3109fc967f35209a6a6f83195 +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..13750f7bf0102c568e70266cc5546d4610ef4536 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79cadc7c2f4bab2fb6e3c440448c6bfcf80e1c0be9bd178e919a42ccf02882de +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/code_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x8/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..1a6a07ebaf52d0a6eb85722fafe6b26b36959f32 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 133.342685, + "recon_r2": 0.9579364822230559, + "l1_per_token": 622.5739052568949 + }, + "seed1": { + "l0": 133.74438666666666, + "recon_r2": 0.9580450849409071, + "l1_per_token": 624.8499954625632 + }, + "seed2": { + "l0": 132.899125, + "recon_r2": 0.9579131916194232, + "l1_per_token": 622.6106457408799 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.5577321366719193, + "mcc_abs": 0.557746485916359, + "act_iou": 0.12213332597887794, + "act_iou_n_pairs": 6144, + "input_r2": 0.9688372132773408, + "matched_mcc_signed": { + "0.6": 0.642655610066397, + "0.7": 0.6531531421587754, + "0.8": 0.6600240988435896, + "0.9": 0.6771347244925351 + }, + "matched_mcc_abs": { + "0.6": 0.6411153437110967, + "0.7": 0.651264146359182, + "0.8": 0.6593221182755177, + "0.9": 0.6763283143523651 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5555783110863869, + "mcc_abs": 0.5555893378922883, + "act_iou": 0.12190028999310154, + "act_iou_n_pairs": 6144, + "input_r2": 0.9688121912263958, + "matched_mcc_signed": { + "0.6": 0.6417433238566577, + "0.7": 0.6537775509211723, + "0.8": 0.6626166110304657, + "0.9": 0.677604842646902 + }, + "matched_mcc_abs": { + "0.6": 0.6406682577340987, + "0.7": 0.6541223723378987, + "0.8": 0.6618975300687223, + "0.9": 0.6771529459612937 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5573900481677448, + "mcc_abs": 0.5574042106974981, + "act_iou": 0.1224582018667922, + "act_iou_n_pairs": 6144, + "input_r2": 0.968922839675886, + "matched_mcc_signed": { + "0.6": 0.6428978422155754, + "0.7": 0.6520874050185033, + "0.8": 0.6611660515592243, + "0.9": 0.6789412539771904 + }, + "matched_mcc_abs": { + "0.6": 0.6407262362148244, + "0.7": 0.6505745275770458, + "0.8": 0.6607224377840826, + "0.9": 0.6784005985357081 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 133.32873222222221, + "l0_std": 0.4228035374613265, + "recon_r2_mean": 0.9579649195944621, + "recon_r2_std": 7.039513571534565e-05, + "l1_per_token_mean": 623.3448488201128, + "l1_per_token_std": 1.3036246688564024, + "input_r2_mean": 0.968857414726541, + "input_r2_std": 5.802450995898652e-05, + "mcc_signed_mean": 0.5569001653086837, + "mcc_signed_std": 0.0011574670945322814, + "mcc_abs_mean": 0.5569133448353818, + "mcc_abs_std": 0.0011593247475929613, + "matched_mcc_signed_per_threshold": { + "0.6": 0.6424322587128768, + "0.7": 0.6530060326994837, + "0.8": 0.6612689204777599, + "0.9": 0.6778936070388758 + }, + "matched_mcc_signed_band": 0.658650204732249, + "matched_mcc_abs_per_threshold": { + "0.6": 0.6408366125533399, + "0.7": 0.6519870154247088, + "0.8": 0.6606473620427742, + "0.9": 0.677293952949789 + }, + "matched_mcc_abs_band": 0.657691235742653, + "act_iou_mean": 0.12216393927959056, + "act_iou_std": 0.00028021294641155596, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/dict_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x8/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..84972e78d27e9cd92625f17df698d6853a895702 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3640, + "0.6": 2442, + "0.7": 1557, + "0.8": 821, + "0.9": 339 + }, + "n_dup_clusters_signed": { + "0.5": 1333, + "0.6": 894, + "0.7": 573, + "0.8": 300, + "0.9": 131 + }, + "largest_dup_signed": { + "0.5": 37, + "0.6": 17, + "0.7": 16, + "0.8": 11, + "0.9": 9 + }, + "babel_mu1_signed": 0.6008203625679016, + "n_dup_abs": { + "0.5": 3655, + "0.6": 2476, + "0.7": 1573, + "0.8": 836, + "0.9": 339 + }, + "n_dup_clusters_abs": { + "0.5": 1335, + "0.6": 908, + "0.7": 580, + "0.8": 306, + "0.9": 131 + }, + "largest_dup_abs": { + "0.5": 37, + "0.6": 17, + "0.7": 16, + "0.8": 11, + "0.9": 9 + }, + "babel_mu1_abs": 0.6029527187347412 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3642, + "0.6": 2486, + "0.7": 1536, + "0.8": 868, + "0.9": 317 + }, + "n_dup_clusters_signed": { + "0.5": 1317, + "0.6": 908, + "0.7": 557, + "0.8": 324, + "0.9": 125 + }, + "largest_dup_signed": { + "0.5": 34, + "0.6": 28, + "0.7": 13, + "0.8": 11, + "0.9": 8 + }, + "babel_mu1_signed": 0.6018854975700378, + "n_dup_abs": { + "0.5": 3663, + "0.6": 2513, + "0.7": 1557, + "0.8": 882, + "0.9": 320 + }, + "n_dup_clusters_abs": { + "0.5": 1324, + "0.6": 919, + "0.7": 566, + "0.8": 331, + "0.9": 126 + }, + "largest_dup_abs": { + "0.5": 34, + "0.6": 30, + "0.7": 14, + "0.8": 11, + "0.9": 8 + }, + "babel_mu1_abs": 0.6040351390838623 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3597, + "0.6": 2418, + "0.7": 1556, + "0.8": 848, + "0.9": 322 + }, + "n_dup_clusters_signed": { + "0.5": 1295, + "0.6": 869, + "0.7": 559, + "0.8": 303, + "0.9": 115 + }, + "largest_dup_signed": { + "0.5": 37, + "0.6": 25, + "0.7": 17, + "0.8": 11, + "0.9": 9 + }, + "babel_mu1_signed": 0.5993590950965881, + "n_dup_abs": { + "0.5": 3615, + "0.6": 2442, + "0.7": 1583, + "0.8": 860, + "0.9": 327 + }, + "n_dup_clusters_abs": { + "0.5": 1300, + "0.6": 879, + "0.7": 573, + "0.8": 310, + "0.9": 118 + }, + "largest_dup_abs": { + "0.5": 45, + "0.6": 22, + "0.7": 17, + "0.8": 11, + "0.9": 9 + }, + "babel_mu1_abs": 0.6011876463890076 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8542901873588562, + "dcs_abs": 0.8564583659172058, + "frac_sign_flipped": 0.017415364583333332, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8426259160041809, + "0.6": 0.8466602563858032, + "0.7": 0.8478602766990662, + "0.8": 0.8523461818695068, + "0.9": 0.8540911078453064 + }, + "unique_frac_signed": { + "0.5": 0.623046875, + "0.6": 0.74560546875, + "0.7": 0.8402506510416666, + "0.8": 0.913330078125, + "0.9": 0.9674479166666666 + }, + "dedup_dcs_signed": { + "0.5": 0.5249954437604174, + "0.6": 0.631274517334532, + "0.7": 0.712415149488758, + "0.8": 0.7784734048764221, + "0.9": 0.8262886629284669 + }, + "matched_frac_signed": { + "0.5": 0.546875, + "0.6": 0.6399739583333334, + "0.7": 0.6829427083333334, + "0.8": 0.6884765625, + "0.9": 0.6181640625 + }, + "matched_cos_abs": { + "0.5": 0.846027672290802, + "0.6": 0.848763644695282, + "0.7": 0.8503178358078003, + "0.8": 0.8545229434967041, + "0.9": 0.8560903072357178 + }, + "unique_frac_abs": { + "0.5": 0.620849609375, + "0.6": 0.74267578125, + "0.7": 0.8385416666666666, + "0.8": 0.9120279947916666, + "0.9": 0.96728515625 + }, + "dedup_dcs_abs": { + "0.5": 0.5252559498621849, + "0.6": 0.630356202920666, + "0.7": 0.7130269352346659, + "0.8": 0.7793488466607716, + "0.9": 0.8280834465986118 + }, + "matched_frac_abs": { + "0.5": 0.54833984375, + "0.6": 0.6394856770833334, + "0.7": 0.68603515625, + "0.8": 0.6884765625, + "0.9": 0.6181640625 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8556812405586243, + "dcs_abs": 0.857814610004425, + "frac_sign_flipped": 0.0166015625, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8444643020629883, + "0.6": 0.847666323184967, + "0.7": 0.8498727083206177, + "0.8": 0.8532578349113464, + "0.9": 0.8539199233055115 + }, + "unique_frac_signed": { + "0.5": 0.6249186197916666, + "0.6": 0.7479654947916666, + "0.7": 0.8387858072916666, + "0.8": 0.9132486979166666, + "0.9": 0.9662272135416666 + }, + "dedup_dcs_signed": { + "0.5": 0.5277214661085357, + "0.6": 0.6340251608392766, + "0.7": 0.7128611657438644, + "0.8": 0.7792366067199812, + "0.9": 0.825080668083198 + }, + "matched_frac_signed": { + "0.5": 0.552734375, + "0.6": 0.6476236979166666, + "0.7": 0.6868489583333334, + "0.8": 0.6847330729166666, + "0.9": 0.6181640625 + }, + "matched_cos_abs": { + "0.5": 0.8471776843070984, + "0.6": 0.8505780100822449, + "0.7": 0.8527635335922241, + "0.8": 0.8553610444068909, + "0.9": 0.8560706377029419 + }, + "unique_frac_abs": { + "0.5": 0.622802734375, + "0.6": 0.7451985677083334, + "0.7": 0.8369954427083334, + "0.8": 0.912109375, + "0.9": 0.966064453125 + }, + "dedup_dcs_abs": { + "0.5": 0.5276245782879414, + "0.6": 0.6338495148374932, + "0.7": 0.7137591913245463, + "0.8": 0.7801828276133165, + "0.9": 0.8270194124488626 + }, + "matched_frac_abs": { + "0.5": 0.5524088541666666, + "0.6": 0.6487630208333334, + "0.7": 0.6884765625, + "0.8": 0.6861979166666666, + "0.9": 0.61865234375 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8569879531860352, + "dcs_abs": 0.8593170046806335, + "frac_sign_flipped": 0.017740885416666668, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8467940092086792, + "0.6": 0.850313127040863, + "0.7": 0.8523238897323608, + "0.8": 0.8527654409408569, + "0.9": 0.8570263385772705 + }, + "unique_frac_signed": { + "0.5": 0.6234537760416666, + "0.6": 0.7455240885416666, + "0.7": 0.8391927083333334, + "0.8": 0.911376953125, + "0.9": 0.967529296875 + }, + "dedup_dcs_signed": { + "0.5": 0.5279369225706129, + "0.6": 0.6339289190121538, + "0.7": 0.7152639934017013, + "0.8": 0.7771907692949753, + "0.9": 0.8291980907670222 + }, + "matched_frac_signed": { + "0.5": 0.5520833333333334, + "0.6": 0.6463216145833334, + "0.7": 0.6930338541666666, + "0.8": 0.6863606770833334, + "0.9": 0.6181640625 + }, + "matched_cos_abs": { + "0.5": 0.8501198291778564, + "0.6": 0.8531087040901184, + "0.7": 0.8549855947494507, + "0.8": 0.8550905585289001, + "0.9": 0.8593066930770874 + }, + "unique_frac_abs": { + "0.5": 0.6212565104166666, + "0.6": 0.7430826822916666, + "0.7": 0.837158203125, + "0.8": 0.910400390625, + "0.9": 0.9672037760416666 + }, + "dedup_dcs_abs": { + "0.5": 0.5281424785110478, + "0.6": 0.6339303041216529, + "0.7": 0.7157582041982096, + "0.8": 0.7784747785044601, + "0.9": 0.8311246783220364 + }, + "matched_frac_abs": { + "0.5": 0.5525716145833334, + "0.6": 0.64794921875, + "0.7": 0.6954752604166666, + "0.8": 0.68798828125, + "0.9": 0.6189778645833334 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.6006883184115092, + "babel_mu1_signed_std": 0.0012683667091544298, + "n_dup_signed_per_threshold": { + "0.5": 3626.3333333333335, + "0.6": 2448.6666666666665, + "0.7": 1549.6666666666667, + "0.8": 845.6666666666666, + "0.9": 326.0 + }, + "n_dup_signed_band": 1292.5, + "n_dup_signed_band_std": 8.227241335952167, + "n_dup_clusters_signed_per_threshold": { + "0.5": 1315.0, + "0.6": 890.3333333333334, + "0.7": 563.0, + "0.8": 309.0, + "0.9": 123.66666666666667 + }, + "n_dup_clusters_signed_band": 471.50000000000006, + "n_dup_clusters_signed_band_std": 8.888194417315589, + "largest_dup_signed_per_threshold": { + "0.5": 36.0, + "0.6": 23.333333333333332, + "0.7": 15.333333333333334, + "0.8": 11.0, + "0.9": 8.666666666666666 + }, + "largest_dup_signed_band": 14.583333333333332, + "largest_dup_signed_band_std": 1.1814539065631522, + "dcs_signed_mean": 0.8556531270345052, + "dcs_signed_std": 0.001349102624803774, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5268846108131887, + "0.6": 0.6330761990619874, + "0.7": 0.7135134362114414, + "0.8": 0.7783002602971262, + "0.9": 0.8268558072595624 + }, + "dedup_dcs_signed_band": 0.7379364257075294, + "dedup_dcs_signed_band_std": 0.0008989495748107668, + "matched_cos_signed_per_threshold": { + "0.5": 0.8446280757586161, + "0.6": 0.8482132355372111, + "0.7": 0.8500189582506815, + "0.8": 0.8527898192405701, + "0.9": 0.8550124565760294 + }, + "matched_cos_signed_band": 0.8515086174011229, + "matched_cos_signed_band_std": 0.0014619767956287413, + "unique_frac_signed_per_threshold": { + "0.5": 0.623806423611111, + "0.6": 0.746365017361111, + "0.7": 0.8394097222222222, + "0.8": 0.9126519097222222, + "0.9": 0.967068142361111 + }, + "unique_frac_signed_band": 0.8663736979166666, + "unique_frac_signed_band_std": 0.0004084240702287405, + "matched_frac_signed_per_threshold": { + "0.5": 0.5505642361111112, + "0.6": 0.6446397569444445, + "0.7": 0.6876085069444445, + "0.8": 0.6865234375, + "0.9": 0.6181640625 + }, + "matched_frac_signed_band": 0.6592339409722223, + "matched_frac_signed_band_std": 0.0017928289541946734, + "babel_mu1_abs_mean": 0.6027251680692037, + "babel_mu1_abs_std": 0.0014373197768251494, + "n_dup_abs_per_threshold": { + "0.5": 3644.3333333333335, + "0.6": 2477.0, + "0.7": 1571.0, + "0.8": 859.3333333333334, + "0.9": 328.6666666666667 + }, + "n_dup_abs_band": 1309.0, + "n_dup_abs_band_std": 7.937253933193772, + "n_dup_clusters_abs_per_threshold": { + "0.5": 1319.6666666666667, + "0.6": 902.0, + "0.7": 573.0, + "0.8": 315.6666666666667, + "0.9": 125.0 + }, + "n_dup_clusters_abs_band": 478.9166666666667, + "n_dup_clusters_abs_band_std": 8.009109397013711, + "largest_dup_abs_per_threshold": { + "0.5": 38.666666666666664, + "0.6": 23.0, + "0.7": 15.666666666666666, + "0.8": 11.0, + "0.9": 8.666666666666666 + }, + "largest_dup_abs_band": 14.583333333333332, + "largest_dup_abs_band_std": 1.2583057392117918, + "dcs_abs_mean": 0.8578633268674215, + "dcs_abs_std": 0.0014299419199736995, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5270076688870581, + "0.6": 0.6327120072932707, + "0.7": 0.7141814435858073, + "0.8": 0.779335484259516, + "0.9": 0.8287425124565035 + }, + "dedup_dcs_abs_band": 0.7387428618987744, + "dedup_dcs_abs_band_std": 0.0010596366557693272, + "matched_cos_abs_per_threshold": { + "0.5": 0.8477750619252523, + "0.6": 0.8508167862892151, + "0.7": 0.8526889880498251, + "0.8": 0.8549915154774984, + "0.9": 0.8571558793385824 + }, + "matched_cos_abs_band": 0.8539132922887803, + "matched_cos_abs_band_std": 0.001610907561081223, + "unique_frac_abs_per_threshold": { + "0.5": 0.6216362847222222, + "0.6": 0.74365234375, + "0.7": 0.8375651041666666, + "0.8": 0.9115125868055555, + "0.9": 0.9668511284722222 + }, + "unique_frac_abs_band": 0.864895290798611, + "unique_frac_abs_band_std": 0.00037642928294214797, + "matched_frac_abs_per_threshold": { + "0.5": 0.5511067708333334, + "0.6": 0.6453993055555556, + "0.7": 0.6899956597222222, + "0.8": 0.6875542534722222, + "0.9": 0.6185980902777778 + }, + "matched_frac_abs_band": 0.6603868272569444, + "matched_frac_abs_band_std": 0.002281671362234943, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/config.json b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b3fd244772bb48fe3d86fad8c03b0af36f99e8b2 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/model.safetensors b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8f76f7fc734b7117511af49de2af3ea854b59441 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54b3e94850029a0a46bfff3e50f3fb6d0387c84ea48732bff3a701dc95c4620a +size 37776696 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b08cd466c0fc441f7a73337549f9fdf1d2d93a20 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f39464ddf437e3449e6d46420fc94b35e4c260d0927e88b014ed85d938e772be +size 50496 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..dff0dc4aea686df670cf6619179697a0d23c4701 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51db8f375b183d1aeb73855da60f6c816c492894103aa3d9e931eb6cbc4f8ac4 +size 25895 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..42e57c584b8debcc03eefad9e1450b434ece69fa --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 14.916806689453125, + "l0_per_token_0.001": 14.916806689453125, + "l0_per_token_0.01": 14.916806689453125, + "l0_per_token_0.05": 14.916806689453125, + "l0_per_token_0.1": 14.916806689453125, + "l0_per_token_0.5": 14.916806689453125, + "l0_per_token_1.0": 14.9143650390625, + "l1_per_token": 265.01689360429793, + "r2_score": 0.8894365591273903, + "l2_loss": 2.5929817816794567 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6209f14801e05486cb3e7dde7c49fc8a752420e3 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce5e0e9cb2c9ead1e49b096475699c24c94b65637a7f162c7fe2a5372c32cb7a +size 25806 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dddcd4fbfbfd6978385e42dc9fe20a9856dd79cd --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fd2ee0fe4307c0d94179cabb6707cf0b120c8ef6b79c8374bb6ed8abec9eacd +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c577a6fb3f080d663222adebd6219daffcf9d10 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5d72fc10deb95387849e64a3f949bc6544540436a895698433c0ca91d98376e +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/config.json b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b57334bdfb7c241cea507faaf65ed199052d3e8f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/model.safetensors b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c71cc3dcc80526a12a63ee2692d7edfe53c6fcd3 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd56808a0cfdc11d9582a46d55cb652a59bc102a1ddb7ec32c63ebcd6ae7ac8 +size 37776696 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2edb01a73dcc85e66f2471c71a1c28f94bbded0c --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ea7cdc1373f6fbe47cdfad71c99448b86d6594e8bfa37716e7177159b7c147d +size 50496 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fcbe1c7974baaf9d6165fb32b731eb7e7b1b03b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3663faa374844d8072a644c6a69dadf2a03e014053cfd7e6c15911eb00163768 +size 25895 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e0e3b01c5070f294f3793715ad6c9a66bd588088 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 15.187090625, + "l0_per_token_0.001": 15.187090625, + "l0_per_token_0.01": 15.187090625, + "l0_per_token_0.05": 15.187090625, + "l0_per_token_0.1": 15.187090625, + "l0_per_token_0.5": 15.187090625, + "l0_per_token_1.0": 15.185013623046874, + "l1_per_token": 267.8888524205817, + "r2_score": 0.8899659762104499, + "l2_loss": 2.5805656625676696 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ad83d80f397eded9c4ef94e9962762bd9870bcb --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f737f4ff74d8038c4d46c5ce56701809047a496a9b844f6fa26bf6eb669ce389 +size 25806 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e116b343c8e74a0774f8e811a1f37b525831296 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d44b766dff5cac3fee2e5306f20dabc603d157432f89cccb1956c664e9ebebfb +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..950f6b3bd8e8e353e0bb4cb4c5616ffadbab2925 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45ecb96c70650646097e184a9d63ae587736459149d7809db40ccb800083429e +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/config.json b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..abd1311d294b93d49cdd3c8a13e2ddb9120dc92d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/model.safetensors b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e64fc257fef848dc974f9f771edcb57d012935c0 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0debc5f746cea79ad38d56a650ae8440f211bdd45f62a8a9444311f3059cbf0f +size 37776696 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7cb685c6fe1be659e22aa49942e695b94e6fe277 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9980ecc1196c6833ff6867ad58ec89b65cac6c1cdc78ddb91a6bf1bc4e872413 +size 50496 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c2497504142f40c430757a8010b9279b9f66432 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0449412d8f787f76bdc76a3c8b1cd4c104038723bfccc7f900079dc6ba1278e2 +size 25895 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..15ca1f9f09b990e6c6da9aa1e2ef1c8beca6f3f3 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 14.739921923828126, + "l0_per_token_0.001": 14.739921923828126, + "l0_per_token_0.01": 14.739921923828126, + "l0_per_token_0.05": 14.739921923828126, + "l0_per_token_0.1": 14.739921923828126, + "l0_per_token_0.5": 14.739921923828126, + "l0_per_token_1.0": 14.738592138671875, + "l1_per_token": 264.7434869000165, + "r2_score": 0.8891176971020024, + "l2_loss": 2.60045987223249 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2740a3900247d39e8fbf2127c06089cbd95c151 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d5f4d1fd929f4536767dd8c1367dd1ee9d52d6995ba5711134b6967c27ee00f +size 25806 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f77b46bf07755d01509d39beb5d00e151d3d47f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:913d5dd80ddb868913dc945914371061307245c3109fc967f35209a6a6f83195 +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..13750f7bf0102c568e70266cc5546d4610ef4536 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79cadc7c2f4bab2fb6e3c440448c6bfcf80e1c0be9bd178e919a42ccf02882de +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/code_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x8/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..1a6a07ebaf52d0a6eb85722fafe6b26b36959f32 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 133.342685, + "recon_r2": 0.9579364822230559, + "l1_per_token": 622.5739052568949 + }, + "seed1": { + "l0": 133.74438666666666, + "recon_r2": 0.9580450849409071, + "l1_per_token": 624.8499954625632 + }, + "seed2": { + "l0": 132.899125, + "recon_r2": 0.9579131916194232, + "l1_per_token": 622.6106457408799 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.5577321366719193, + "mcc_abs": 0.557746485916359, + "act_iou": 0.12213332597887794, + "act_iou_n_pairs": 6144, + "input_r2": 0.9688372132773408, + "matched_mcc_signed": { + "0.6": 0.642655610066397, + "0.7": 0.6531531421587754, + "0.8": 0.6600240988435896, + "0.9": 0.6771347244925351 + }, + "matched_mcc_abs": { + "0.6": 0.6411153437110967, + "0.7": 0.651264146359182, + "0.8": 0.6593221182755177, + "0.9": 0.6763283143523651 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5555783110863869, + "mcc_abs": 0.5555893378922883, + "act_iou": 0.12190028999310154, + "act_iou_n_pairs": 6144, + "input_r2": 0.9688121912263958, + "matched_mcc_signed": { + "0.6": 0.6417433238566577, + "0.7": 0.6537775509211723, + "0.8": 0.6626166110304657, + "0.9": 0.677604842646902 + }, + "matched_mcc_abs": { + "0.6": 0.6406682577340987, + "0.7": 0.6541223723378987, + "0.8": 0.6618975300687223, + "0.9": 0.6771529459612937 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5573900481677448, + "mcc_abs": 0.5574042106974981, + "act_iou": 0.1224582018667922, + "act_iou_n_pairs": 6144, + "input_r2": 0.968922839675886, + "matched_mcc_signed": { + "0.6": 0.6428978422155754, + "0.7": 0.6520874050185033, + "0.8": 0.6611660515592243, + "0.9": 0.6789412539771904 + }, + "matched_mcc_abs": { + "0.6": 0.6407262362148244, + "0.7": 0.6505745275770458, + "0.8": 0.6607224377840826, + "0.9": 0.6784005985357081 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 133.32873222222221, + "l0_std": 0.4228035374613265, + "recon_r2_mean": 0.9579649195944621, + "recon_r2_std": 7.039513571534565e-05, + "l1_per_token_mean": 623.3448488201128, + "l1_per_token_std": 1.3036246688564024, + "input_r2_mean": 0.968857414726541, + "input_r2_std": 5.802450995898652e-05, + "mcc_signed_mean": 0.5569001653086837, + "mcc_signed_std": 0.0011574670945322814, + "mcc_abs_mean": 0.5569133448353818, + "mcc_abs_std": 0.0011593247475929613, + "matched_mcc_signed_per_threshold": { + "0.6": 0.6424322587128768, + "0.7": 0.6530060326994837, + "0.8": 0.6612689204777599, + "0.9": 0.6778936070388758 + }, + "matched_mcc_signed_band": 0.658650204732249, + "matched_mcc_abs_per_threshold": { + "0.6": 0.6408366125533399, + "0.7": 0.6519870154247088, + "0.8": 0.6606473620427742, + "0.9": 0.677293952949789 + }, + "matched_mcc_abs_band": 0.657691235742653, + "act_iou_mean": 0.12216393927959056, + "act_iou_std": 0.00028021294641155596, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/dict_metrics_step_final.json b/lm/gpt2-small_layer8/mp/x8/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..84972e78d27e9cd92625f17df698d6853a895702 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3640, + "0.6": 2442, + "0.7": 1557, + "0.8": 821, + "0.9": 339 + }, + "n_dup_clusters_signed": { + "0.5": 1333, + "0.6": 894, + "0.7": 573, + "0.8": 300, + "0.9": 131 + }, + "largest_dup_signed": { + "0.5": 37, + "0.6": 17, + "0.7": 16, + "0.8": 11, + "0.9": 9 + }, + "babel_mu1_signed": 0.6008203625679016, + "n_dup_abs": { + "0.5": 3655, + "0.6": 2476, + "0.7": 1573, + "0.8": 836, + "0.9": 339 + }, + "n_dup_clusters_abs": { + "0.5": 1335, + "0.6": 908, + "0.7": 580, + "0.8": 306, + "0.9": 131 + }, + "largest_dup_abs": { + "0.5": 37, + "0.6": 17, + "0.7": 16, + "0.8": 11, + "0.9": 9 + }, + "babel_mu1_abs": 0.6029527187347412 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3642, + "0.6": 2486, + "0.7": 1536, + "0.8": 868, + "0.9": 317 + }, + "n_dup_clusters_signed": { + "0.5": 1317, + "0.6": 908, + "0.7": 557, + "0.8": 324, + "0.9": 125 + }, + "largest_dup_signed": { + "0.5": 34, + "0.6": 28, + "0.7": 13, + "0.8": 11, + "0.9": 8 + }, + "babel_mu1_signed": 0.6018854975700378, + "n_dup_abs": { + "0.5": 3663, + "0.6": 2513, + "0.7": 1557, + "0.8": 882, + "0.9": 320 + }, + "n_dup_clusters_abs": { + "0.5": 1324, + "0.6": 919, + "0.7": 566, + "0.8": 331, + "0.9": 126 + }, + "largest_dup_abs": { + "0.5": 34, + "0.6": 30, + "0.7": 14, + "0.8": 11, + "0.9": 8 + }, + "babel_mu1_abs": 0.6040351390838623 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3597, + "0.6": 2418, + "0.7": 1556, + "0.8": 848, + "0.9": 322 + }, + "n_dup_clusters_signed": { + "0.5": 1295, + "0.6": 869, + "0.7": 559, + "0.8": 303, + "0.9": 115 + }, + "largest_dup_signed": { + "0.5": 37, + "0.6": 25, + "0.7": 17, + "0.8": 11, + "0.9": 9 + }, + "babel_mu1_signed": 0.5993590950965881, + "n_dup_abs": { + "0.5": 3615, + "0.6": 2442, + "0.7": 1583, + "0.8": 860, + "0.9": 327 + }, + "n_dup_clusters_abs": { + "0.5": 1300, + "0.6": 879, + "0.7": 573, + "0.8": 310, + "0.9": 118 + }, + "largest_dup_abs": { + "0.5": 45, + "0.6": 22, + "0.7": 17, + "0.8": 11, + "0.9": 9 + }, + "babel_mu1_abs": 0.6011876463890076 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8542901873588562, + "dcs_abs": 0.8564583659172058, + "frac_sign_flipped": 0.017415364583333332, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8426259160041809, + "0.6": 0.8466602563858032, + "0.7": 0.8478602766990662, + "0.8": 0.8523461818695068, + "0.9": 0.8540911078453064 + }, + "unique_frac_signed": { + "0.5": 0.623046875, + "0.6": 0.74560546875, + "0.7": 0.8402506510416666, + "0.8": 0.913330078125, + "0.9": 0.9674479166666666 + }, + "dedup_dcs_signed": { + "0.5": 0.5249954437604174, + "0.6": 0.631274517334532, + "0.7": 0.712415149488758, + "0.8": 0.7784734048764221, + "0.9": 0.8262886629284669 + }, + "matched_frac_signed": { + "0.5": 0.546875, + "0.6": 0.6399739583333334, + "0.7": 0.6829427083333334, + "0.8": 0.6884765625, + "0.9": 0.6181640625 + }, + "matched_cos_abs": { + "0.5": 0.846027672290802, + "0.6": 0.848763644695282, + "0.7": 0.8503178358078003, + "0.8": 0.8545229434967041, + "0.9": 0.8560903072357178 + }, + "unique_frac_abs": { + "0.5": 0.620849609375, + "0.6": 0.74267578125, + "0.7": 0.8385416666666666, + "0.8": 0.9120279947916666, + "0.9": 0.96728515625 + }, + "dedup_dcs_abs": { + "0.5": 0.5252559498621849, + "0.6": 0.630356202920666, + "0.7": 0.7130269352346659, + "0.8": 0.7793488466607716, + "0.9": 0.8280834465986118 + }, + "matched_frac_abs": { + "0.5": 0.54833984375, + "0.6": 0.6394856770833334, + "0.7": 0.68603515625, + "0.8": 0.6884765625, + "0.9": 0.6181640625 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8556812405586243, + "dcs_abs": 0.857814610004425, + "frac_sign_flipped": 0.0166015625, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8444643020629883, + "0.6": 0.847666323184967, + "0.7": 0.8498727083206177, + "0.8": 0.8532578349113464, + "0.9": 0.8539199233055115 + }, + "unique_frac_signed": { + "0.5": 0.6249186197916666, + "0.6": 0.7479654947916666, + "0.7": 0.8387858072916666, + "0.8": 0.9132486979166666, + "0.9": 0.9662272135416666 + }, + "dedup_dcs_signed": { + "0.5": 0.5277214661085357, + "0.6": 0.6340251608392766, + "0.7": 0.7128611657438644, + "0.8": 0.7792366067199812, + "0.9": 0.825080668083198 + }, + "matched_frac_signed": { + "0.5": 0.552734375, + "0.6": 0.6476236979166666, + "0.7": 0.6868489583333334, + "0.8": 0.6847330729166666, + "0.9": 0.6181640625 + }, + "matched_cos_abs": { + "0.5": 0.8471776843070984, + "0.6": 0.8505780100822449, + "0.7": 0.8527635335922241, + "0.8": 0.8553610444068909, + "0.9": 0.8560706377029419 + }, + "unique_frac_abs": { + "0.5": 0.622802734375, + "0.6": 0.7451985677083334, + "0.7": 0.8369954427083334, + "0.8": 0.912109375, + "0.9": 0.966064453125 + }, + "dedup_dcs_abs": { + "0.5": 0.5276245782879414, + "0.6": 0.6338495148374932, + "0.7": 0.7137591913245463, + "0.8": 0.7801828276133165, + "0.9": 0.8270194124488626 + }, + "matched_frac_abs": { + "0.5": 0.5524088541666666, + "0.6": 0.6487630208333334, + "0.7": 0.6884765625, + "0.8": 0.6861979166666666, + "0.9": 0.61865234375 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8569879531860352, + "dcs_abs": 0.8593170046806335, + "frac_sign_flipped": 0.017740885416666668, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8467940092086792, + "0.6": 0.850313127040863, + "0.7": 0.8523238897323608, + "0.8": 0.8527654409408569, + "0.9": 0.8570263385772705 + }, + "unique_frac_signed": { + "0.5": 0.6234537760416666, + "0.6": 0.7455240885416666, + "0.7": 0.8391927083333334, + "0.8": 0.911376953125, + "0.9": 0.967529296875 + }, + "dedup_dcs_signed": { + "0.5": 0.5279369225706129, + "0.6": 0.6339289190121538, + "0.7": 0.7152639934017013, + "0.8": 0.7771907692949753, + "0.9": 0.8291980907670222 + }, + "matched_frac_signed": { + "0.5": 0.5520833333333334, + "0.6": 0.6463216145833334, + "0.7": 0.6930338541666666, + "0.8": 0.6863606770833334, + "0.9": 0.6181640625 + }, + "matched_cos_abs": { + "0.5": 0.8501198291778564, + "0.6": 0.8531087040901184, + "0.7": 0.8549855947494507, + "0.8": 0.8550905585289001, + "0.9": 0.8593066930770874 + }, + "unique_frac_abs": { + "0.5": 0.6212565104166666, + "0.6": 0.7430826822916666, + "0.7": 0.837158203125, + "0.8": 0.910400390625, + "0.9": 0.9672037760416666 + }, + "dedup_dcs_abs": { + "0.5": 0.5281424785110478, + "0.6": 0.6339303041216529, + "0.7": 0.7157582041982096, + "0.8": 0.7784747785044601, + "0.9": 0.8311246783220364 + }, + "matched_frac_abs": { + "0.5": 0.5525716145833334, + "0.6": 0.64794921875, + "0.7": 0.6954752604166666, + "0.8": 0.68798828125, + "0.9": 0.6189778645833334 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.6006883184115092, + "babel_mu1_signed_std": 0.0012683667091544298, + "n_dup_signed_per_threshold": { + "0.5": 3626.3333333333335, + "0.6": 2448.6666666666665, + "0.7": 1549.6666666666667, + "0.8": 845.6666666666666, + "0.9": 326.0 + }, + "n_dup_signed_band": 1292.5, + "n_dup_signed_band_std": 8.227241335952167, + "n_dup_clusters_signed_per_threshold": { + "0.5": 1315.0, + "0.6": 890.3333333333334, + "0.7": 563.0, + "0.8": 309.0, + "0.9": 123.66666666666667 + }, + "n_dup_clusters_signed_band": 471.50000000000006, + "n_dup_clusters_signed_band_std": 8.888194417315589, + "largest_dup_signed_per_threshold": { + "0.5": 36.0, + "0.6": 23.333333333333332, + "0.7": 15.333333333333334, + "0.8": 11.0, + "0.9": 8.666666666666666 + }, + "largest_dup_signed_band": 14.583333333333332, + "largest_dup_signed_band_std": 1.1814539065631522, + "dcs_signed_mean": 0.8556531270345052, + "dcs_signed_std": 0.001349102624803774, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5268846108131887, + "0.6": 0.6330761990619874, + "0.7": 0.7135134362114414, + "0.8": 0.7783002602971262, + "0.9": 0.8268558072595624 + }, + "dedup_dcs_signed_band": 0.7379364257075294, + "dedup_dcs_signed_band_std": 0.0008989495748107668, + "matched_cos_signed_per_threshold": { + "0.5": 0.8446280757586161, + "0.6": 0.8482132355372111, + "0.7": 0.8500189582506815, + "0.8": 0.8527898192405701, + "0.9": 0.8550124565760294 + }, + "matched_cos_signed_band": 0.8515086174011229, + "matched_cos_signed_band_std": 0.0014619767956287413, + "unique_frac_signed_per_threshold": { + "0.5": 0.623806423611111, + "0.6": 0.746365017361111, + "0.7": 0.8394097222222222, + "0.8": 0.9126519097222222, + "0.9": 0.967068142361111 + }, + "unique_frac_signed_band": 0.8663736979166666, + "unique_frac_signed_band_std": 0.0004084240702287405, + "matched_frac_signed_per_threshold": { + "0.5": 0.5505642361111112, + "0.6": 0.6446397569444445, + "0.7": 0.6876085069444445, + "0.8": 0.6865234375, + "0.9": 0.6181640625 + }, + "matched_frac_signed_band": 0.6592339409722223, + "matched_frac_signed_band_std": 0.0017928289541946734, + "babel_mu1_abs_mean": 0.6027251680692037, + "babel_mu1_abs_std": 0.0014373197768251494, + "n_dup_abs_per_threshold": { + "0.5": 3644.3333333333335, + "0.6": 2477.0, + "0.7": 1571.0, + "0.8": 859.3333333333334, + "0.9": 328.6666666666667 + }, + "n_dup_abs_band": 1309.0, + "n_dup_abs_band_std": 7.937253933193772, + "n_dup_clusters_abs_per_threshold": { + "0.5": 1319.6666666666667, + "0.6": 902.0, + "0.7": 573.0, + "0.8": 315.6666666666667, + "0.9": 125.0 + }, + "n_dup_clusters_abs_band": 478.9166666666667, + "n_dup_clusters_abs_band_std": 8.009109397013711, + "largest_dup_abs_per_threshold": { + "0.5": 38.666666666666664, + "0.6": 23.0, + "0.7": 15.666666666666666, + "0.8": 11.0, + "0.9": 8.666666666666666 + }, + "largest_dup_abs_band": 14.583333333333332, + "largest_dup_abs_band_std": 1.2583057392117918, + "dcs_abs_mean": 0.8578633268674215, + "dcs_abs_std": 0.0014299419199736995, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5270076688870581, + "0.6": 0.6327120072932707, + "0.7": 0.7141814435858073, + "0.8": 0.779335484259516, + "0.9": 0.8287425124565035 + }, + "dedup_dcs_abs_band": 0.7387428618987744, + "dedup_dcs_abs_band_std": 0.0010596366557693272, + "matched_cos_abs_per_threshold": { + "0.5": 0.8477750619252523, + "0.6": 0.8508167862892151, + "0.7": 0.8526889880498251, + "0.8": 0.8549915154774984, + "0.9": 0.8571558793385824 + }, + "matched_cos_abs_band": 0.8539132922887803, + "matched_cos_abs_band_std": 0.001610907561081223, + "unique_frac_abs_per_threshold": { + "0.5": 0.6216362847222222, + "0.6": 0.74365234375, + "0.7": 0.8375651041666666, + "0.8": 0.9115125868055555, + "0.9": 0.9668511284722222 + }, + "unique_frac_abs_band": 0.864895290798611, + "unique_frac_abs_band_std": 0.00037642928294214797, + "matched_frac_abs_per_threshold": { + "0.5": 0.5511067708333334, + "0.6": 0.6453993055555556, + "0.7": 0.6899956597222222, + "0.8": 0.6875542534722222, + "0.9": 0.6185980902777778 + }, + "matched_frac_abs_band": 0.6603868272569444, + "matched_frac_abs_band_std": 0.002281671362234943, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/config.json b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b3fd244772bb48fe3d86fad8c03b0af36f99e8b2 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/model.safetensors b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8f76f7fc734b7117511af49de2af3ea854b59441 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54b3e94850029a0a46bfff3e50f3fb6d0387c84ea48732bff3a701dc95c4620a +size 37776696 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b08cd466c0fc441f7a73337549f9fdf1d2d93a20 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f39464ddf437e3449e6d46420fc94b35e4c260d0927e88b014ed85d938e772be +size 50496 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..dff0dc4aea686df670cf6619179697a0d23c4701 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51db8f375b183d1aeb73855da60f6c816c492894103aa3d9e931eb6cbc4f8ac4 +size 25895 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..42e57c584b8debcc03eefad9e1450b434ece69fa --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 14.916806689453125, + "l0_per_token_0.001": 14.916806689453125, + "l0_per_token_0.01": 14.916806689453125, + "l0_per_token_0.05": 14.916806689453125, + "l0_per_token_0.1": 14.916806689453125, + "l0_per_token_0.5": 14.916806689453125, + "l0_per_token_1.0": 14.9143650390625, + "l1_per_token": 265.01689360429793, + "r2_score": 0.8894365591273903, + "l2_loss": 2.5929817816794567 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6209f14801e05486cb3e7dde7c49fc8a752420e3 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce5e0e9cb2c9ead1e49b096475699c24c94b65637a7f162c7fe2a5372c32cb7a +size 25806 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dddcd4fbfbfd6978385e42dc9fe20a9856dd79cd --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fd2ee0fe4307c0d94179cabb6707cf0b120c8ef6b79c8374bb6ed8abec9eacd +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c577a6fb3f080d663222adebd6219daffcf9d10 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5d72fc10deb95387849e64a3f949bc6544540436a895698433c0ca91d98376e +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/config.json b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b57334bdfb7c241cea507faaf65ed199052d3e8f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/model.safetensors b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c71cc3dcc80526a12a63ee2692d7edfe53c6fcd3 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd56808a0cfdc11d9582a46d55cb652a59bc102a1ddb7ec32c63ebcd6ae7ac8 +size 37776696 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2edb01a73dcc85e66f2471c71a1c28f94bbded0c --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ea7cdc1373f6fbe47cdfad71c99448b86d6594e8bfa37716e7177159b7c147d +size 50496 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fcbe1c7974baaf9d6165fb32b731eb7e7b1b03b --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3663faa374844d8072a644c6a69dadf2a03e014053cfd7e6c15911eb00163768 +size 25895 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e0e3b01c5070f294f3793715ad6c9a66bd588088 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 15.187090625, + "l0_per_token_0.001": 15.187090625, + "l0_per_token_0.01": 15.187090625, + "l0_per_token_0.05": 15.187090625, + "l0_per_token_0.1": 15.187090625, + "l0_per_token_0.5": 15.187090625, + "l0_per_token_1.0": 15.185013623046874, + "l1_per_token": 267.8888524205817, + "r2_score": 0.8899659762104499, + "l2_loss": 2.5805656625676696 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ad83d80f397eded9c4ef94e9962762bd9870bcb --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f737f4ff74d8038c4d46c5ce56701809047a496a9b844f6fa26bf6eb669ce389 +size 25806 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e116b343c8e74a0774f8e811a1f37b525831296 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d44b766dff5cac3fee2e5306f20dabc603d157432f89cccb1956c664e9ebebfb +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..950f6b3bd8e8e353e0bb4cb4c5616ffadbab2925 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45ecb96c70650646097e184a9d63ae587736459149d7809db40ccb800083429e +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/config.json b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..abd1311d294b93d49cdd3c8a13e2ddb9120dc92d --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/model.safetensors b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e64fc257fef848dc974f9f771edcb57d012935c0 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0debc5f746cea79ad38d56a650ae8440f211bdd45f62a8a9444311f3059cbf0f +size 37776696 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7cb685c6fe1be659e22aa49942e695b94e6fe277 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9980ecc1196c6833ff6867ad58ec89b65cac6c1cdc78ddb91a6bf1bc4e872413 +size 50496 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c2497504142f40c430757a8010b9279b9f66432 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0449412d8f787f76bdc76a3c8b1cd4c104038723bfccc7f900079dc6ba1278e2 +size 25895 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..15ca1f9f09b990e6c6da9aa1e2ef1c8beca6f3f3 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 14.739921923828126, + "l0_per_token_0.001": 14.739921923828126, + "l0_per_token_0.01": 14.739921923828126, + "l0_per_token_0.05": 14.739921923828126, + "l0_per_token_0.1": 14.739921923828126, + "l0_per_token_0.5": 14.739921923828126, + "l0_per_token_1.0": 14.738592138671875, + "l1_per_token": 264.7434869000165, + "r2_score": 0.8891176971020024, + "l2_loss": 2.60045987223249 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2740a3900247d39e8fbf2127c06089cbd95c151 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d5f4d1fd929f4536767dd8c1367dd1ee9d52d6995ba5711134b6967c27ee00f +size 25806 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f77b46bf07755d01509d39beb5d00e151d3d47f --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:913d5dd80ddb868913dc945914371061307245c3109fc967f35209a6a6f83195 +size 25811 diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..13750f7bf0102c568e70266cc5546d4610ef4536 --- /dev/null +++ b/lm/gpt2-small_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79cadc7c2f4bab2fb6e3c440448c6bfcf80e1c0be9bd178e919a42ccf02882de +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..36db3f73b2d76e762e3ab5a6936e2897c9127ba7 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bcb3ea3351462d1118c04127dcf065880e80f93c460cad0f8ea6070ea4528f7 +size 75550008 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb0533a95768c47bedb4f4c7e93155d03594caab --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d389e5a6b206de15f433ee3a05101ef497d519db738279cc6db13fbdd77df6b +size 99648 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..800ad8749425353476bd5b994184d4b4516c9832 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2d71c748caa07e96d725369a857c31fac0ecb8ff6f73183f4a3f01f2ace3780 +size 50471 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3129366155296d4b1b969fcc52a8b0605fb21e4 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0df2e31c6d1c251ba949b3372f5fc38e217bad7616b48a3427c5fb642f4afa2 +size 50382 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..edf46f6597fd327a56a46a93ebddb96dcf48cee8 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfd9c8c714213aeb1d9b4a7073ac1683279ed91463b5403ee45cc0f1e376b0cf +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..502e727230aa5ebd8547f5d12de87cb5ada4bf0d --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76034a81f245a79952dd4f6c542b0d5b7829832637f30e9542b8122c1fa2f017 +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ae43bffb634c6445d7488feb7c0eb0e98c10aa5d --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4825714cbc5a0b1126562ffc15683c27df694e3981e2df159742c11ae52626e5 +size 75550008 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe849aa1d43cd54931f733aac8b2fc80db22bb95 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25dcd836581830d53747d9ddfca7803cfcab23106e802eefc11a279fc79330c4 +size 99648 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e7c2cb9f2105de80c8b7ca787debf0727abb96d --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce50755729081559a6b9351e2b0facefe0ff119feaa6552b66091194fe3cf43d +size 50471 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5eddb11e8c56e42a03220f53195842cf16e75b83 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fda8207c8a3d7da2f899ee90355b4eb228bd2138158ed8ccfe9872d9571a344 +size 50382 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..239afd2c55b76e0c94519752d53afd8876f36afe --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69995f6b4b5d9c403711bbd9838d0049a65a7a96587569d9fa2b50c49530f9d2 +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..48b91ed1129a61800cab443dd2e2f9f4a37bd1e1 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:179b3a785ae68946919a483a85b4b59c10e5b23dc913b51fed9974d778921b1b +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/model.safetensors b/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..33fbd785f528c15b6dd541554c04467324f5beb5 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a4e4b998819fdf02fe7b7f65b8aa483511b61d4fb61185f49536bfde07440c3 +size 75550008 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7c7d1a1c9641ed4d01fb89f36b3973c2f7694b2 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10827f5948ca88a574c41b5551a3c80146cba160968a224620ba84d41b40c1b3 +size 99648 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..1451a0e112e3b753715059ad1c6711a106a3c620 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ad0fd8f60bba242bf9c53f0222e08f1fa0d837ad8b6f3cccec1afbb1559b167 +size 50471 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0edc0713107c89693b19f15691c0fa733bfdffa --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0552e68a5bd1a222593603fd12518a6470de8d38d049e69dfa05cd295744c919 +size 50382 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..722ed5c36dcb4560226ab3854f4b59be326163e1 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80b838fc14de7fb820c409338dac5030b371217dae1f73ff9d15ecd7af037150 +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d68aa357c9b761fcf7c2c13b98ece4b30eebc01a --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:259286cfd31e2032244d10bb7e8768a70b836bd8446831459b4db08b00fb11ea +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/model.safetensors b/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..aa5ba49de60d0963c84beecd48d73487b65f8726 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:630397c20ff003b6a625b3f1232f090c370fa8c993c06a8a211ead3f851f381f +size 75550008 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0306c64decdae3aba765d9980d30bf7020e52380 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:508518a9e1a6129185683cfa3b9fc00e213bfc54163872e9d75e1567e288d92d +size 99648 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..458e7226cf657d6d8623cde39b4a4f68556727d2 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38becf614ace856fe99c65beda734c7edc2df07b889076442cac9a52e4ef5134 +size 50471 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c349f580575f4243ef987bd9d21e36c7d9ac32f8 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15241c4b761945ddcb7efda69ed6cf667a7263221b0b22a7a67d68161a56a6d8 +size 50382 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..590913133310078acfcf0a8f62734b8a88055287 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1414751fc16768401d1ce63d6fbad7cf980c0e6b356b32fcd73cd3deae146c78 +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..27cd92d51eab73ac690eb41cc250a28b03148683 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db0023c54d280911da1d6fa5b95528120e36c625e723fed581a6dd122598e4b8 +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/model.safetensors b/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1852618d92bea92161b75a9c0aed5f13ec181f12 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:613dcc3aeed935ee23247fa0ec78da7c9b14a6e18787c8abc9f4a19b72d71087 +size 75550008 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..4979b562d2a5ac529ba83eef73dbd88e70d585f5 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be2491653e0249529e5043eeb0eba23bc4f8feb8eaecbb5cccb9521038087db9 +size 99648 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..3697aad863bd17182f1712824fc32203d96900cd --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14a6f4d3a42941ad273eab547e695fd10a5de233ac3cc52c304e6822dfed84d3 +size 50471 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e484741e55683f42a3dd69a63f4fd7b54f89e86 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6795b8b85665ade3629cd0fed0a68606c185c9e33665e44be2be69df9a8bf71 +size 50382 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9114cd4400a1dffa9b4a122174acf8642cc90cd8 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce8e03ffbf42f97e7a942411e56d1cdd26882ab59d858de0a3ffab0e533543cd +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..57d117b7257fe7e23139bf3972dbf5c19a36cb7d --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92c8385858106e28835d4c66f7ec1e89609bc5e193837a932efd963d80f2790a +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/model.safetensors b/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e6b29e463eb3ce7a8eab6b0c5de74ab8812994f8 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d80fee03496a137c5e8be976de011afe447b6c6646d3a110ca9843cd9318027b +size 75550008 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a37b40cbff7cccb7529fc91e0bbed230a361289 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:264a7e3f19be992b9e80832ef677b46891fa5f40c6344f61a8e30c3442240451 +size 99648 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c6aad681c5a2d8a7a596261da26f03707dc7085 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:241476bea7617b5e103e159ecba2943db1bbae2e57574370f1cb12784eae3e38 +size 50471 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7b9fcb5506795443c3ac57406328a218b227175 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fea7df9c76284e03abaff7759476a599b92a491b1882772b2c583cba664f6897 +size 50382 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc5903b861f6a52aa9c3c34e13d8d2b36936f979 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:206891902f6c2e0612b64e21ff762229221eef719bc15bada0b165190e1a9de2 +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d802b5a49cacaa0909448810083846888b442540 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b7dce4f9fe5e77512728bd2b14821e8ff6d16e8f97a0a4d204b96e8cce933d5 +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/model.safetensors b/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cc590bbf0027d7da28fcddbd35c2748db5ee2210 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73ae72916173e24c903419d983e076e88ed939364bc83eb3fb6daf6b119c8fb2 +size 75550008 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ac8de224d11cb06782ef492350a747b27a08284 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5702be79657c92727a86d8547187672ea2e58772b9e63526b17f594674a9c453 +size 99648 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..257aa4d9dc37f8c9c83c25f2ca6ada1d601e43b6 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:651269a5950bea7a862eb2c21812abf42ab45ba64564f0f95f4153cdd4eaa8b9 +size 50471 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..03f75d79736d137ba0ceac2dcec137f917ffc3f9 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abbbf2d516337232a2a07f3d631a5c8744a4894575e18628e2a7147a4b090f55 +size 50382 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..040c72971f2852f1a6e4c77a85d53a52c57ad9f1 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47ef55ffc20b172b3d5b0dc549b3947e662593cabed7a4ae7ec49214584af97b +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..54e42611a2fcd973e3da4ada6090b0fd7a503821 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24297f8650e5056c06686ac6fe77baa3ac9682a79f35d4bf543b05e387c24dc0 +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/model.safetensors b/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b82483cf674462fdadc783cd73ef587ae76c9a7f --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:251d7f9918fde06df9829a3b4eef45f02364c673abaee5ed62bdec1629d207b4 +size 75550008 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a21be9f79c35dfd630c09d23e94e99cb016683c --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c25f54c3359e44c9b21fd057272fa44e0a94a185c5ff5b0c0624cc99a79fa79e +size 99648 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..533570812418b378467327b5e1ed420bf2abbf47 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20ef3b66370fddb08aaf3f33e9da7d0ecadbc7f6c700014e1a5d292e79ef6e2c +size 50471 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d57960d0c26c7f567676dd7bf0faebd55b5764f8 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c9fe9ac41b5e783d8f2a9f05f382051af0e8bff36f20eb1e7d7e09d324e9877 +size 50382 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fd0bcbf4582dc2f68a53423690cd5ba69fcde48 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00292675b1592eb2ce85cb0e92978bd85b07d4eaaf079d4db093226186948071 +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c93950626444cb2da344871f711b27d0155fc431 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:460b2d1ea32ec2d57ac9597e6b9e9194581033e69f6fec73e3a44c8b630059c4 +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/model.safetensors b/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bc44247436185aa9090efcf565f832b2cd5ecbef --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5330f846504b64a891eef8f63d7359b5bb66c5923e39ef643afdc11872aeb385 +size 75550008 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd0fe22c7518921d7337a9877c4bef44e5e99a4a --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba3340b1592466d7f3689a94e2a9cacb1751044d73f791c089d206002bc2956b +size 99648 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..42557791fb0bce59f2abc3a08193b64e36db7342 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a712b478de8a555976410f24b41bd58d072569c16afb82f18ce2678e56f75093 +size 50471 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..779306e2acaef1995cdbb04701dadb398228502a --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9492c1d63ce64d8751d785d1fd9e3215d8c2f8554ca07c9fa13e58dbcb22457 +size 50382 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7643c20fcfd2594bfe71281937231b2c06c9d2a1 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82a884eebb267753730f68456434e429068dff1e02381d40795aa9eacbacd699 +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8290dd842608ffc7bb6613017765aeb4fbb46355 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4055513e4377e501c228b501061b5b7b0191d23d47ab82baf3c607f51fc53568 +size 50387 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2bee512c5061a4c7c29021d2eccb86cf591f3b47 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6687010a02bd6682abaa9ce675fa802abf8721d7ba9854d9b2cd37e22cb971db +size 18890032 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..922ee107f6b3d47673f606ac5b2e6668c60973f5 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:602c4d59873909dabb9a3807f22a7f11a85b92f71933705f3f62ab65907b80b4 +size 25920 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..495730795150a023eafcd10d0f08b39a181a220d --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fb9bf863a36cb0cd603cee8f3021fb0aba9b6bf84a3c37310c7a7856e52ea92 +size 13607 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea980d00a06d94521c0c4921a226cb2b0d602e35 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4741dc67e592009d87693d4636442d77fa0702a0071b3f17860dba876878ae8d +size 13518 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9cb64a03529dbb5a5c1fc5ebd16fb99955960185 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2429420d9fd65571676783ee5f47fcda960c0cffe2c63ef5d45ce32030b9e5b +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b086ab0f960a1bf5a7c479a708d566ed401d63e --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47d5bdf25a7141957e41980ec207dab88420a191e2732d736113c3e2398945df +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dbf845ec70c1ef0439e9165e2cb21f0407d4e7fc --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ce6b769eab429b429e4a2918012698d4d125a4d997cd0777fbb5ac4e8e4e4ce +size 18890032 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a9a26db09a4b9f0468497b7d3867eacb9703c66 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26cafabe7f80b9195f22fd0e99905b2c1a3e5696b4feb3ab1371ea0a8d7e4ff7 +size 25920 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..61e7df6b631ef234c279b9e4600dd2a022035671 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb8629d694aa7b47f97f46d51a106d125319eabe45e6b7a6e6e17416c0f890f0 +size 13607 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8328f301c1518da85c75714fa1666b60eae4975b --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3942b95e88d8d4628fc99a5f7e350b8723080ce1c5c6de4d62bb85d08cdfcfb5 +size 13518 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a920eef16fbd9b5438a9018395d6e8a9f5deb26 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9deece4d1ce804f89afa0ccad74f1819b8895109b758be35fbac90c477ff068e +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7deaecee1aad7e3a442002a1c08ff7657d2e91d5 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cd4902caa2e48dc86a2b1cf1c0657a5359eebb0b9ef7442fb4837ed19b84fee +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/model.safetensors b/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..42b9dc057ba73e9c8142a55aafd7c5b8dbf73ce7 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e353d8c6e2746b57bda277a82717efd3dbd9a649096f4bd9011693088ccee31 +size 18890032 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c695ad5354336fad588a9bc779f6c61e72142bf --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80f3c7c098650def7389776e6db7e931376c88b862efb68e4df94cf1fb642698 +size 25920 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c62209f9af9068718387b95b2a5e3cbcfca3a00 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e506257af1f6dcbb32619c34e555aacdf488f523a7e6818ac158758acd615f30 +size 13607 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8b96c004db559f9f637da281e2b4d19d2dd46f7 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30de33c15ddd53da4ee6a29c0a6869eeb60af16fbc7d20c2f967b419413183de +size 13518 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a762ef9bd003970d23b549ce343a9dd5ae8a0e8 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f934b844eb1980dd6c100e8eae9cfb0cd044b620b90539ce83052c268c028f1 +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..19270a8b03c1480daa3769154b145722dd7577ed --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:418c264c4310a4095a6c35adfb43772e0a300ea70c8bbfc347620829791141de +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/model.safetensors b/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f84a1cef88b669f3b38e66aef3d796f1df189557 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f999f8cda1a94d28c1e4b857b40f0a11f8827df7af694ac3d87cc28a82b1e144 +size 18890032 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..22cdbaa8e9a08e1d34ddb0511a39c438531e814e --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ece4b46698f79cc93543ba3b41973694cc63aba1527708258eb892c2ed25a2e +size 25920 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..565217721e614551e4f044f600afb72607d06519 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc8e7f18d738b5fd158e9cee0f3854ca6a5f1ac18079720ecce5cebd9f9e5d9f +size 13607 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f65dd7b6f1710793dfe90558b37ddbf360d4448 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5434d32c9a174901fabf7955bd85b78640cda060f255b2b98b971556015eb3fb +size 13518 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e28ac0d17bf5f55f191a099f22bd1cf32e70e381 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85fac5c47b313e19d9ec5432f8ff1b695b35862e4313c4ef287959dbc31128e4 +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2cab69b56d46671a96e7b87df6f51a21d692cb3 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ae7cdc69479d322ad82da96275fc2e83054f7e9c92a2cf93f14933fe7c45157 +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/model.safetensors b/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9ee430817b8cce6d71411a7b954803e472c0aa31 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2fe25811408219f48ecc3e57345ab2dfde1edb487da436600f520d1ebcf05ba +size 18890032 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2dc515a0bb9675418436e58ca4b1614aaa8165a2 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b5a28dd70a5649d85896df94422372f7d4b5f3b8c69b07c13347d061fdcc9e5 +size 25920 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a06ca581915e89a88bf9578caaa13b2fd498545 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9eb3b5aca8a5bf178a29765731e67e41b3231e6a49b54a334efb70c67e6dfdb +size 13607 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee6277ec9e7774fd01677a24d04206f53ba6502c --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20eaa1266f71e4b30f863fa032aa8ccc73ef17e491afc5964db58c12cad4e4e1 +size 13518 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..89b5785a6f17c7a987de683dd2b9090674489ae7 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b71f6336c579ccd6440190ec4673f4811139fe82cfecca0db08975ddad95387f +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..06ed6bd7f5623cfe40df89a97cdbecedc8ca6751 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56bd3fc7a74a84c162ed8116832a02941971038e5d6b2d3fbc007e0b44eeb51c +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/model.safetensors b/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..49cc62a87fd2e22eb32e5ff21713ef8672f0efc7 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef1909dfedc8e6f7b4f37817e5d622085e43d0cfb057937dcdc9377007c34bcf +size 18890032 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5d25529e27d5d7bc4b052ce9acaa5b21f71fa3a --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f0f131ee22ac67de224bc85dec96414c6ff83be598554702a13b866a3f2e892 +size 25920 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..57692e189675450883b4c81b651728f3c52b72a2 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c088b012b6ebb247b5c13ebaf44fb7b2a510ac347dce1001a439aac9c479234 +size 13607 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c4f23b23cd2ec90e27180791ddd3f9cb5d91c322 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36e416cfeaeddd38d962f797d3ed75a8b94a6329cf2d0fa72a1155177754702f +size 13518 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e31066d08174214fa95a9e901ce48024ddb636f --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42d092d89b6ce171152b5980aa8839ab00aa92c3c6b9b3037fa51deea2624351 +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..786306b0b78a54a05a907aeff2da68f2f42ac8ae --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b0e757dc9af8bc2edada0cacd64c63c937d13744588b03fd359b6bb26f487c0 +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/model.safetensors b/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ac518a56b7cc06c1bda7a12cb5dc1f77c65e92cb --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48d6208e9ab3bddc0dc4fb5df2ebdab1c1821a934bb99ee707c64befec1c961e +size 18890032 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f80887cb22be500b7ee851ef5d8a20a48dadde8a --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5d58e8d59a47fc18902411d4420ac896d21424a3e67fbd3913e0fc0d02ccb16 +size 25920 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..b774a61f75789ff492e225625281262839a1186d --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91d78184dc5c9278c2ac76b8d1859923f877fe6276fbc08577018a2f1e3de39b +size 13607 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..711719ada6288ebee284d1b734ed1c971d1fd832 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dec5c653b797698497625de009df472ac2cde87fe5274da30a442ca792dc6db9 +size 13518 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..df79df23ee1e88519943dc94aaed03981864bbee --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba26e351cb36f74142dd417822ce20c9f38fb7ef6d1d9e5a68c93659ac836fad +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9ad6637b14ee8f85e19781ed644d88a8c29fe6d --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:616a5bddfac85841920e8c464960cc9173cedb80c78aed34af292ff15fc70b97 +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/model.safetensors b/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a9b18f2aeac60746fac5f9b3c3ecdfcb928694ad --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f7d6c0a555a73b03969315b53ddc0186d2558e50e0341fad419a2f388641382 +size 18890032 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b473b9ca026388663b5d331cabd885a15d1b1358 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe27caf069e33339f4e0a05a4db45edccb54f5023745c2b28d6635719c4d1476 +size 25920 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..713a3eb41fd13b663512181ae8f5aeebc41d7d67 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e96da12cb4fbd6439367051c878a74b439e07e2b71e2a22c294a58ade8955c58 +size 13607 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..687c7102fd0bfa77bd3d09b31ee017416c6f1992 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0096d2c76d2a60795c88d86c986f375dcad3de2d0cf303c1a53ef7e08ec94d5d +size 13518 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..715b20953b96d4f3acd0c8be8b8ee63dcd88f680 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9574adab2edc09d3ce1b9ca68c508555be580938be9094b865c5bc7aab6f6fa9 +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..610c63fbdd6c7ec045ca2c15febf98865a348efc --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20271d81cca0539e37657e180da4d042235b684116c66d282fe93493f8ed6ea1 +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/model.safetensors b/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ffad951135fcab2deac9176e0dfee02d62e67f0f --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85b9137f161b9549d18137565fe0e10f2e1acd4bd7e7e58e51d2522d6f58cae7 +size 18890032 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b094aeae133f8d00548e22b22b0aa4f2af8426bc --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f10ef376ece51ff9ac00e12d9bb9928fdecd9b91fc896e8f22a7156a0f776f6b +size 25920 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..fef68dcedeab4cbff503cbcb50f3dfca61b6b378 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9c4ee958bf5d0860c3772002aa895e8ff49fe963b1c1e2e87a8d1d806ffc8cc +size 13607 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ac2a343b04ea89bdeda2577cf041107dfdd515c --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7445a798b8616818f42f9c9a3b0770d4c5a87932c170b220617d8fd7a30ea7f5 +size 13518 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b2cd497422e4ced1cbeaae57d25fd5a0eefc9fa --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d386ad31d10efe974929303e99b474bac2a15da2d4e9d5f5706cb6e5c2fb778d +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..4cfb9e5e1c8e89fea11d0b7b88b735f9dbd5d553 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60a5836396260abb410b98e360bd0d8c90551d3460d08181e2bb216e8e53a520 +size 13523 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/config.json b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..03af2764551fb0c668e41544170a64c51da927c1 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/config.json @@ -0,0 +1,10 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "relu", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "class_type": "VanillaSAE", + "module_name": "fastsae.models.sae" +} diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/model.safetensors b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8ca2c9c1d9c0c2c480bf904c31da1a74744b79fe --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd5445b178cf6e3789b341fad229e83914c69703bf4a68649476a285f045730b +size 37776696 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4354db47350411d2ac1b3ccd552cf2f52ada2a0 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:371e14a5e1d160e466c07eb3e7ceb0ecd74278a07602a30396bc72d96c5e1027 +size 50496 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1feedb3c9eb9866f281f3c3ae45aec4d31c555c --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:330523580a807bbb589247a9ecb9322f691974cbe0ea082621de4c871fb95edd +size 25895 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..7756a46866b1d5c4d328d60d8d3508d1c7111bbb --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 123.205973828125, + "l0_per_token_0.001": 123.12448935546875, + "l0_per_token_0.01": 122.39314018554687, + "l0_per_token_0.05": 119.20439497070312, + "l0_per_token_0.1": 115.35157192382813, + "l0_per_token_0.5": 89.25716108398437, + "l0_per_token_1.0": 65.9142162109375, + "l1_per_token": 267.0475748861746, + "r2_score": 0.9196522611171748, + "l2_loss": 1.8843500299737583 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ea5d92f827453686069a9349c44e7b87299a201 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9c392a735f843c9c961f92dcfcacc4a953e802f5ecc9684aa91e44a3c5ba603 +size 25806 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3513e66e5fce1eb1d60158017c7a02e5a00af51 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a9203428081236f145f0b7970818bd64573fe7c1a917d8a3eadd0af6c2f000f +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7a5559ed516b1aafda7b8d9e354c2a644cb7800 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e7dbe8cdc287369ea32aad3e18bbc60439de70b763ce2e49d515b3e556edc11 +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/config.json b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2df92766c911a9cf15a36365f5f0d91bde86a2e6 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/config.json @@ -0,0 +1,10 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "relu", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "class_type": "VanillaSAE", + "module_name": "fastsae.models.sae" +} diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/model.safetensors b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9a6139a2ac1e928d9794cd183411ba18f64217ce --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c57f45026756317f313c3767c1e1d2c52f65991ce0dd62a203223e9727c81a1 +size 37776696 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8adf11bd6670fd00e80a6a8979c60008dd476b1c --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9d4104f1c27b4fd56d28d8eb98569968400764fe46af98128f0a164ac9d71d2 +size 50496 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..40e268bf6c15cd37ddfec8b27008fa12cc33c3d1 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7eaffb6f9a8f9f691584bbc174dfd395d1aeae8896249792a08d51d5c3067223 +size 25895 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..41f2f11205aa1ccf06781f573d3ccef04d920cb8 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 124.420975, + "l0_per_token_0.001": 124.33897783203125, + "l0_per_token_0.01": 123.60356142578125, + "l0_per_token_0.05": 120.39413193359375, + "l0_per_token_0.1": 116.51679619140624, + "l0_per_token_0.5": 90.23810581054687, + "l0_per_token_1.0": 66.68841362304687, + "l1_per_token": 269.44816389988114, + "r2_score": 0.9202786477365921, + "l2_loss": 1.8696597392264924 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dddfb65c504d63c4a1ac5762c12d9ba0fc61d38f --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:497726e841fcd2059f1f4a9a63d9b4f71740657977113e455ea03326460b3518 +size 25806 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..039c6de8db0670f2a98191052443534e994542e0 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:704ff561daecf66c0fcebdae7ce2201ad5e735bdd535424b3769ed7fc1708483 +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf53821176ec1757ff3a51e59ddb9c8ef54126e7 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed25451d53a86adcf2d0e9f417858f1a521d8437fad15e00e4f3f046a4721366 +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/config.json b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6ff526fe86b1fee16310adf1b05c81222284af8c --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/config.json @@ -0,0 +1,10 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "relu", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "class_type": "VanillaSAE", + "module_name": "fastsae.models.sae" +} diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/model.safetensors b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7bca16b182a5cfdcad79b56118efa2a0b2671b1f --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f74aed3f89efc52a3764c701dfdaec3e2dcc2afe4b55671d52535871deaaceb +size 37776696 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..13fe433c8cf8fa08ae38bd09fed09be8eef89c68 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:651572f2d16183cbbcdaee06ae6d5cfc98bc7788e797a0fa057fe9435750b601 +size 50496 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a12bcc1790fa9e28b30fbd9e1a5151f639f4241 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:008b83e52c3368a761bf741ae73f08a7ac0799bd039f44c4483e9d87a579faaf +size 25895 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..c3bbc14d0104629932f43396c92f2b522f9627c1 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 123.91108662109374, + "l0_per_token_0.001": 123.82892954101563, + "l0_per_token_0.01": 123.09142705078125, + "l0_per_token_0.05": 119.87450356445312, + "l0_per_token_0.1": 115.98837392578125, + "l0_per_token_0.5": 89.66872211914063, + "l0_per_token_1.0": 66.13663510742188, + "l1_per_token": 267.3776820182874, + "r2_score": 0.9197673443730127, + "l2_loss": 1.8816510475306576 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..11f98dee670781e4a233a6636ead9033b707eef9 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c66f90ad75695a424da413861f14167401367bbb121ebff70ab9c88c28eb4b0 +size 25806 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3bad2d112f06106b08e3e86a6fed5004660b8d95 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7af7dcd66c8385e29d93cfd2a790cbd54f6c367ddbd02c9c1fe812889a534ea6 +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a6ec54d53935511a007ddb36675d3d56e2ada02 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e50f5dd2efe53b639cbf97f0d55298fe7524775197a20346e695c9bd25984d79 +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/config.json b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..03af2764551fb0c668e41544170a64c51da927c1 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/config.json @@ -0,0 +1,10 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "relu", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "class_type": "VanillaSAE", + "module_name": "fastsae.models.sae" +} diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/model.safetensors b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0813b1a89ce581338ebd8fede0a32cfc834718ce --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:245b39e5f58d09a296a055a6e01826c9b4233d03093caac8de86b3355306608f +size 37776696 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e8d6de78870aa9a6dc25612f0c4dfa0f16c5a60 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7598c1d928cc7b89a3b24c0f90153ad6e52ef706a2059e611371bd2cea9a9a7 +size 50496 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..806b31340ed36285e45e3b2bf68bcc6cb9f31797 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb1573ebafccedb9608d0e343b5f2602efe4ffa1ec13647dc4a2faea782e717d +size 25895 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..748b17bfd544b48fca61062dd1395cdf639255d0 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 33.19051518554687, + "l0_per_token_0.001": 33.18045317382813, + "l0_per_token_0.01": 33.090155078125, + "l0_per_token_0.05": 32.69280102539062, + "l0_per_token_0.1": 32.20477768554687, + "l0_per_token_0.5": 28.624292724609376, + "l0_per_token_1.0": 24.839003466796875, + "l1_per_token": 158.90994628863464, + "r2_score": 0.870863341941793, + "l2_loss": 3.0285689288352606 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6dc45071fe31099b1410ce67a279658d79cc2198 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70b09bf6c7ac0e2e79391f1abc759877c41e1c19bf60bd77186165fc9b341d26 +size 25806 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7339bd4b9e256537e1c08d20877473943efdf0bf --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b02191a28ed14a1afa600c51777f3095a24329a939670878f350741c9ac30a67 +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5eb67854a50ecd12df12e88612c9c5f2d227c138 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f492170191c4d8099c8e9051adb0ddfe0b228c37b65390a15b6eafa8b3ee1815 +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/config.json b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2df92766c911a9cf15a36365f5f0d91bde86a2e6 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/config.json @@ -0,0 +1,10 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "relu", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "class_type": "VanillaSAE", + "module_name": "fastsae.models.sae" +} diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/model.safetensors b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ef13355fbfffe9bf96e468d9d54ec8ef1a4d8e61 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19f0dc19b99d1656dd3e063362da80ccbc36c08b30bc46047186d57a65f680c3 +size 37776696 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0e6aac36fee846116c2688fab8e99e28912a22e --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12148167c1285a3ca182300d01dda5cb5f8eaf2486e4a9ebab3a6cb8250697a3 +size 50496 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..becc72c662aee529e51049b9f151974727532432 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:880a3f70c98b454c3c02ee7967949625fd38ef71c9b62715c101e1fbcea4397b +size 25895 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..a88e659e91f6a8bbecad832d60b3cc86d05f365d --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 33.65359697265625, + "l0_per_token_0.001": 33.643274072265626, + "l0_per_token_0.01": 33.55016337890625, + "l0_per_token_0.05": 33.140535986328125, + "l0_per_token_0.1": 32.63733500976563, + "l0_per_token_0.5": 28.94413642578125, + "l0_per_token_1.0": 25.041516796875, + "l1_per_token": 158.003167719045, + "r2_score": 0.8731963945779327, + "l2_loss": 2.97385316625168 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..82ff257a7593da6fcfb9601cef1be9fd3773123b --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22badd1858500d445d90437e2dfa4894408c45e4c3564c76265c99f15d9ecf50 +size 25806 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..587875217a3e5ed3c73655138c2b278e0435e452 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35e9fac14c78ee17d7c4b26058a16e6047c4e25bab6b9943c1e13ba27eba6794 +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d191ef7b3f9f5e61023f6d2b91c7f1698b21b506 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e64f124a705d0008935974fb0df55eef420eae2d443cab4935ba038f4c748026 +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/config.json b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6ff526fe86b1fee16310adf1b05c81222284af8c --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/config.json @@ -0,0 +1,10 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "relu", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "class_type": "VanillaSAE", + "module_name": "fastsae.models.sae" +} diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/model.safetensors b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..460fa2506f7df2fee4f17b8ac2c1f52dbb9da86e --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8edbf5264122304fe874feb9d05bb6bf301e1bd84d7df1e5f854e8f5f4d5abf +size 37776696 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1332978cfa1effd976093ffc59f891cbc67ae61 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cbc919358e11d8898aa929b2e87aef8709e029a83500400c79c3ddb13de2585 +size 50496 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..b44d696e834ffc1cf288c8ee88d14bf228f4e12f --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3305096b99d2818c4b9bb20ed8940098d5459ae70208c44b2058d14f2da107ab +size 25895 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..27d153c62149650755dae9f6aeedacd282e19ea7 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 33.137987158203124, + "l0_per_token_0.001": 33.12794775390625, + "l0_per_token_0.01": 33.037772216796874, + "l0_per_token_0.05": 32.640116845703126, + "l0_per_token_0.1": 32.151931591796874, + "l0_per_token_0.5": 28.56732255859375, + "l0_per_token_1.0": 24.77502626953125, + "l1_per_token": 157.82196205586865, + "r2_score": 0.8702214027804207, + "l2_loss": 3.0436239645437 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a2f4b4da70775e0c885ed06956d9a4eb2135b14 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99b66fb3f002618eab8d201ed5fc6ece8e0fd74365dfe4cff949170c944deded +size 25806 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d391b865f4f48b5497b560574e54e676c065ea45 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b284a671b3d0637d555e737961878ae13619e4bc0435ecab1d62c3dea6b5ddd9 +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c4a13f88aeb7a6cbcac73e2e22717ceb8f06e381 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87481fe0a09ab0f6b09e6b6a6ab4b5a05c91ca387a1c3604089d34171b1361c3 +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/model.safetensors b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e760220161f93973d14f70ac49698072b5132b57 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4284ce6f8f9a892d7df911e5f5a64cc36b59648b0bbf55459361afe66aad5299 +size 37776696 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ceb72587271cb49d9845c4ad1c699596f5ace3f9 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8195ef8ac8a4feeaea41fb7d5c6adacc2268fc670160578c051a1a5d3a848068 +size 50496 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6f8ed38368ddcc4b56ba2221dfc3690111013a1 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58f039405fed083c16ecb1d9b8af8017ba2ba3add510d4f3eb15d9475a2948fa +size 25895 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..fb2d26fd3e359a6a1c33233164e2a1c32322309c --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 67.09376083984375, + "l0_per_token_0.001": 67.062530859375, + "l0_per_token_0.01": 66.78198017578126, + "l0_per_token_0.05": 65.55192885742187, + "l0_per_token_0.1": 64.05317241210938, + "l0_per_token_0.5": 53.443653955078126, + "l0_per_token_1.0": 43.0256962890625, + "l1_per_token": 207.49205339636524, + "r2_score": 0.9006148371493142, + "l2_loss": 2.330823973876547 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3bbcdcbc1b366ad8489152df88c2ff8cd6d73e1b --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:193fe9c9a8fc2cd492080b37364112e86955dbf3bb6da62bc5ac12543a2ee27d +size 25806 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1feacedfc91113d597d7492a14d3d04b1d2a18ba --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a81ed93308ecabfd3b6a76984e0d089670a23b2598b782486d7f2d2f1988706 +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2c0a5064f792122d134f0eaed3c124b9083ed8c --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59f1d0534da2df440f94d9e608bc772aa03685a930c1e5e0d489955d48a3fe1f +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/model.safetensors b/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1238ffed92f72db9a42ef08c06d0c3e26e28fc57 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d753f02bf30accfb10db25d8363c7ba68476ee8d70d08652e292b305d1e71d7c +size 37776696 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d95350e33d38ae1763752bac2a0178b3aff9c61c --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d231aa95b3680b39012553572985ea0989b611871626188e43aa9d433f41ac0 +size 50496 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..89df2d5cf7c658d66120f44f2b699e1fc0579683 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:032c491f4e8ebb3dbf4563169a687b6c8fad48e9c11c3e5706a6c208f6541663 +size 25895 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..67a54fb0948330e97cc541453fc1c060978f4ad5 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1597ff2bd51c0e2e2bf21fa3086dd39b2f3278bbd84e810911a1ca5eeeec86ac +size 25806 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3af3cc1e43276dc77fc7ae84f0b6ebd2b595addc --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7380be8a4dd06389797872f8eb8a2ff6dda8d53efb933b649c3d53dd4545670 +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..14f3a19ca89f7f1fe551a35146d5618a4950e8ac --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e91b067cb4948a860390d364090fd091edefeca586b540a0eab633d0fb9c64fe +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/model.safetensors b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fc75f510d3e7c2b39276c9a7335621585aacd380 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fd36e743569b8dbefcfd811a373c2cd8fdeafbe812242c5881be687e398894f +size 37776696 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0792a920206240fe175f586cb40f73f41814100a --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21360c9687580220ab439ea0ee90f777ab42e16603f8784d6845c215adc86de1 +size 50496 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..e202c326f2034978af1e7de99e3687462051bd47 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:461abdcf1732409895830fb2c8bbc30f3f35e42eca82430303abc0d2c6c29f40 +size 25895 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..2b8ca188473c41bea6fbc7ee6f3c363761cc2ba4 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 66.92253725585938, + "l0_per_token_0.001": 66.89123374023437, + "l0_per_token_0.01": 66.60993837890625, + "l0_per_token_0.05": 65.377483203125, + "l0_per_token_0.1": 63.87574135742187, + "l0_per_token_0.5": 53.24973901367188, + "l0_per_token_1.0": 42.82117114257812, + "l1_per_token": 206.50177240079768, + "r2_score": 0.9245507933825287, + "l2_loss": 2.338940620534663 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6c96653da828def2778575b3b8e1e3ad4ac2f17 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b875847a1aa2d2135367fbbd5a3daf323e4e67b8eb93f7cf4ac7e4439867f5f +size 25806 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..276e1350aed07f525b324b0d1b2466727a793b4c --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cffb787b0dd8cd0dd2eae3487b239403775654752ac380a0adf33662fd893fa +size 25811 diff --git a/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..87927c120ffbb0ca6bfe88bae1f1a094660d7170 --- /dev/null +++ b/lm/gpt2-small_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cf82ea2387495e3a1837500e68ec39cd760347291935090e4adc360b1150ccf +size 25811