diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_128/code_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x16/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..a47b871a3b0046245086de87ed7880ba342e360a --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 126.42603333333334, + "recon_r2": 0.943795649182259, + "l1_per_token": 289.4212340388216 + }, + "seed1": { + "l0": 125.811965, + "recon_r2": 0.9438612213988956, + "l1_per_token": 289.53429910061254 + }, + "seed2": { + "l0": 126.05112333333334, + "recon_r2": 0.9438580555624618, + "l1_per_token": 289.6181956334526 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7859704474411874, + "mcc_abs": 0.7859711613635518, + "act_iou": 0.45132704906101173, + "act_iou_n_pairs": 12288, + "input_r2": 0.9913803291545396, + "matched_mcc_signed": { + "0.6": 0.8342782215099661, + "0.7": 0.8634063066238307, + "0.8": 0.8990084994065084, + "0.9": 0.9396039340469318 + }, + "matched_mcc_abs": { + "0.6": 0.8335566129871208, + "0.7": 0.8630353633737552, + "0.8": 0.8988650385049328, + "0.9": 0.9395716929205603 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7828689400311742, + "mcc_abs": 0.7828697764077752, + "act_iou": 0.4478816509665113, + "act_iou_n_pairs": 12288, + "input_r2": 0.991334920205118, + "matched_mcc_signed": { + "0.6": 0.8309814935810688, + "0.7": 0.8613655575710586, + "0.8": 0.8984125478906685, + "0.9": 0.9399642265869411 + }, + "matched_mcc_abs": { + "0.6": 0.8310063654716346, + "0.7": 0.8610575981792729, + "0.8": 0.8984265823137941, + "0.9": 0.9399718561318022 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.783658469940665, + "mcc_abs": 0.7836587034705734, + "act_iou": 0.4499608370893986, + "act_iou_n_pairs": 12288, + "input_r2": 0.9913863229725707, + "matched_mcc_signed": { + "0.6": 0.8351966666901895, + "0.7": 0.8621449988701696, + "0.8": 0.8981497940397523, + "0.9": 0.939461465954832 + }, + "matched_mcc_abs": { + "0.6": 0.8341974655594866, + "0.7": 0.8619405907398374, + "0.8": 0.8981406120242229, + "0.9": 0.939461465954832 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 126.09637388888889, + "l0_std": 0.30952494097287203, + "recon_r2_mean": 0.9438383087145388, + "recon_r2_std": 3.6978134090712164e-05, + "l1_per_token_mean": 289.5245762576289, + "l1_per_token_std": 0.09884011177990036, + "input_r2_mean": 0.9913671907774094, + "input_r2_std": 2.810736274817572e-05, + "mcc_signed_mean": 0.7841659524710088, + "mcc_signed_std": 0.0016118284469957588, + "mcc_abs_mean": 0.7841665470806335, + "mcc_abs_std": 0.00161185483533366, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8334854605937414, + "0.7": 0.8623056210216863, + "0.8": 0.8985236137789764, + "0.9": 0.939676542196235 + }, + "matched_mcc_signed_band": 0.8834978093976598, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8329201480060807, + "0.7": 0.8620111840976218, + "0.8": 0.89847741094765, + "0.9": 0.9396683383357316 + }, + "matched_mcc_abs_band": 0.883269270346771, + "act_iou_mean": 0.4497231790389739, + "act_iou_std": 0.0017349504370717912, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_128/dict_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x16/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..8de1ffae9e7c84e19d9bef9081eff6cb20dfa4d8 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_128/dict_metrics_step_final.json @@ -0,0 +1,550 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 126, + "l0_raw": 126.4695044921875, + "recon_r2": 0.9436933468059865, + "l1_per_token": 289.3836344397387, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 19, + "dense_rate": 0.0015462239583333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3965, + "0.6": 19, + "0.7": 3, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1804, + "0.6": 7, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 8, + "0.6": 4, + "0.7": 3, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.4519537389278412, + "n_dup_abs": { + "0.5": 4019, + "0.6": 117, + "0.7": 19, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 1831, + "0.6": 56, + "0.7": 9, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 8, + "0.6": 4, + "0.7": 3, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_abs": 0.45511364936828613 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 126, + "l0_raw": 125.89824155273438, + "recon_r2": 0.9437663728355269, + "l1_per_token": 289.51439305719344, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 19, + "dense_rate": 0.0015462239583333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3819, + "0.6": 14, + "0.7": 2, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1739, + "0.6": 4, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 7, + "0.6": 7, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.4506072700023651, + "n_dup_abs": { + "0.5": 3913, + "0.6": 124, + "0.7": 10, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 1787, + "0.6": 59, + "0.7": 5, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 7, + "0.6": 7, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_abs": 0.45388028025627136 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 126, + "l0_raw": 126.127159765625, + "recon_r2": 0.9437536036438203, + "l1_per_token": 289.6085860579297, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 20, + "dense_rate": 0.0016276041666666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3873, + "0.6": 21, + "0.7": 2, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1766, + "0.6": 8, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 8, + "0.6": 5, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.45112404227256775, + "n_dup_abs": { + "0.5": 3969, + "0.6": 125, + "0.7": 12, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 1813, + "0.6": 60, + "0.7": 6, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 8, + "0.6": 5, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_abs": 0.4543568193912506 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8446399569511414, + "dcs_abs": 0.8448325991630554, + "frac_sign_flipped": 0.00390625, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8421996831893921, + "0.6": 0.8446352481842041, + "0.7": 0.8446696400642395, + "0.8": 0.8446400165557861, + "0.9": 0.8446400165557861 + }, + "unique_frac_signed": { + "0.5": 0.8274332682291666, + "0.6": 0.9991048177083334, + "0.7": 0.9998779296875, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6968640363629675, + "0.6": 0.8438791456671122, + "0.7": 0.8445665309773176, + "0.8": 0.8446400165557861, + "0.9": 0.8446400165557861 + }, + "matched_frac_signed": { + "0.5": 0.772705078125, + "0.6": 0.9130045572916666, + "0.7": 0.8390299479166666, + "0.8": 0.719970703125, + "0.9": 0.5185546875 + }, + "matched_cos_abs": { + "0.5": 0.8424644470214844, + "0.6": 0.8453661203384399, + "0.7": 0.8450070023536682, + "0.8": 0.8448325991630554, + "0.9": 0.8448325991630554 + }, + "unique_frac_abs": { + "0.5": 0.824462890625, + "0.6": 0.994873046875, + "0.7": 0.9993896484375, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.6945806732401252, + "0.6": 0.8410319678660017, + "0.7": 0.8444912510094582, + "0.8": 0.8448325991630554, + "0.9": 0.8448325991630554 + }, + "matched_frac_abs": { + "0.5": 0.7703450520833334, + "0.6": 0.908935546875, + "0.7": 0.8392740885416666, + "0.8": 0.7202962239583334, + "0.9": 0.518798828125 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8424331545829773, + "dcs_abs": 0.8425608277320862, + "frac_sign_flipped": 0.0032552083333333335, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8382132649421692, + "0.6": 0.8423312306404114, + "0.7": 0.8424346446990967, + "0.8": 0.8424331545829773, + "0.9": 0.8424331545829773 + }, + "unique_frac_signed": { + "0.5": 0.8263346354166666, + "0.6": 0.9989827473958334, + "0.7": 0.9998779296875, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6926446526874012, + "0.6": 0.8414743670024716, + "0.7": 0.8423318084387574, + "0.8": 0.8424331545829773, + "0.9": 0.8424331545829773 + }, + "matched_frac_signed": { + "0.5": 0.77197265625, + "0.6": 0.9117024739583334, + "0.7": 0.83349609375, + "0.8": 0.7119140625, + "0.9": 0.5091959635416666 + }, + "matched_cos_abs": { + "0.5": 0.8384155631065369, + "0.6": 0.8430929183959961, + "0.7": 0.8426738381385803, + "0.8": 0.8425607681274414, + "0.9": 0.8425607681274414 + }, + "unique_frac_abs": { + "0.5": 0.8232421875, + "0.6": 0.994873046875, + "0.7": 0.9993489583333334, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.6902190622058697, + "0.6": 0.8387704205233604, + "0.7": 0.8421252223585423, + "0.8": 0.8425607681274414, + "0.9": 0.8425607681274414 + }, + "matched_frac_abs": { + "0.5": 0.7692057291666666, + "0.6": 0.9073079427083334, + "0.7": 0.8330891927083334, + "0.8": 0.711669921875, + "0.9": 0.5091145833333334 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.843366801738739, + "dcs_abs": 0.8435181975364685, + "frac_sign_flipped": 0.0035807291666666665, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8385906219482422, + "0.6": 0.8433858156204224, + "0.7": 0.8433728814125061, + "0.8": 0.843366801738739, + "0.9": 0.843366801738739 + }, + "unique_frac_signed": { + "0.5": 0.8296305338541666, + "0.6": 0.9990641276041666, + "0.7": 0.9999186197916666, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6957203853720177, + "0.6": 0.8425965141165458, + "0.7": 0.8433042475517141, + "0.8": 0.843366801738739, + "0.9": 0.843366801738739 + }, + "matched_frac_signed": { + "0.5": 0.773681640625, + "0.6": 0.9065755208333334, + "0.7": 0.8380533854166666, + "0.8": 0.7194010416666666, + "0.9": 0.5166829427083334 + }, + "matched_cos_abs": { + "0.5": 0.8398551940917969, + "0.6": 0.8440368175506592, + "0.7": 0.8436010479927063, + "0.8": 0.8435182571411133, + "0.9": 0.8435182571411133 + }, + "unique_frac_abs": { + "0.5": 0.8257649739583334, + "0.6": 0.9947102864583334, + "0.7": 0.9995524088541666, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.6935230024779837, + "0.6": 0.8395721045671962, + "0.7": 0.843223459633009, + "0.8": 0.8435182571411133, + "0.9": 0.8435182571411133 + }, + "matched_frac_abs": { + "0.5": 0.7705891927083334, + "0.6": 0.904541015625, + "0.7": 0.8382161458333334, + "0.8": 0.7191569010416666, + "0.9": 0.5166829427083334 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 19, + "n_dense_std": 0.5773502691896258, + "l0_mean": 126, + "l0_std": 0.287502117670696, + "recon_r2_mean": 0.9437377744284445, + "recon_r2_std": 3.900158063678982e-05, + "l1_per_token_mean": 289.5022045182873, + "l1_per_token_std": 0.1129700313896754, + "babel_mu1_signed_mean": 0.4512283504009247, + "babel_mu1_signed_std": 0.0006792678271853955, + "n_dup_signed_per_threshold": { + "0.5": 3885.6666666666665, + "0.6": 18.0, + "0.7": 2.3333333333333335, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 5.083333333333333, + "n_dup_signed_band_std": 0.9464847243000456, + "n_dup_clusters_signed_per_threshold": { + "0.5": 1769.6666666666667, + "0.6": 6.333333333333333, + "0.7": 1.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 1.8333333333333333, + "n_dup_clusters_signed_band_std": 0.5204164998665332, + "largest_dup_signed_per_threshold": { + "0.5": 7.666666666666667, + "0.6": 5.333333333333333, + "0.7": 2.3333333333333335, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 2.4166666666666665, + "largest_dup_signed_band_std": 0.2886751345948129, + "dcs_signed_mean": 0.8434799710909525, + "dcs_signed_std": 0.0011077453000324367, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.6950763581407955, + "0.6": 0.8426500089287098, + "0.7": 0.8434008623225964, + "0.8": 0.8434799909591675, + "0.9": 0.8434799909591675 + }, + "dedup_dcs_signed_band": 0.8432527132924102, + "dedup_dcs_signed_band_std": 0.001134584968802106, + "matched_cos_signed_per_threshold": { + "0.5": 0.8396678566932678, + "0.6": 0.8434507648150126, + "0.7": 0.8434923887252808, + "0.8": 0.8434799909591675, + "0.9": 0.8434799909591675 + }, + "matched_cos_signed_band": 0.843475783864657, + "matched_cos_signed_band_std": 0.0011226214664995392, + "unique_frac_signed_per_threshold": { + "0.5": 0.8277994791666666, + "0.6": 0.9990505642361112, + "0.7": 0.9998914930555555, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9997355143229166, + "unique_frac_signed_band_std": 1.7619331945452135e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.7727864583333334, + "0.6": 0.9104275173611112, + "0.7": 0.8368598090277777, + "0.8": 0.7170952690972222, + "0.9": 0.5148111979166666 + }, + "matched_frac_signed_band": 0.7447984483506944, + "matched_frac_signed_band_std": 0.003049202308208975, + "babel_mu1_abs_mean": 0.45445024967193604, + "babel_mu1_abs_std": 0.0006219700592711387, + "n_dup_abs_per_threshold": { + "0.5": 3967.0, + "0.6": 122.0, + "0.7": 13.666666666666666, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_abs_band": 33.916666666666664, + "n_dup_abs_band_std": 0.38188130791298663, + "n_dup_clusters_abs_per_threshold": { + "0.5": 1810.3333333333333, + "0.6": 58.333333333333336, + "0.7": 6.666666666666667, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_abs_band": 16.25, + "n_dup_clusters_abs_band_std": 0.25, + "largest_dup_abs_per_threshold": { + "0.5": 7.666666666666667, + "0.6": 5.333333333333333, + "0.7": 2.3333333333333335, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_abs_band": 2.4166666666666665, + "largest_dup_abs_band_std": 0.2886751345948129, + "dcs_abs_mean": 0.84363720814387, + "dcs_abs_std": 0.0011405520602530892, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6927742459746596, + "0.6": 0.8397914976521861, + "0.7": 0.8432799776670032, + "0.8": 0.84363720814387, + "0.9": 0.84363720814387 + }, + "dedup_dcs_abs_band": 0.8425864729017323, + "dedup_dcs_abs_band_std": 0.0011517895005007416, + "matched_cos_abs_per_threshold": { + "0.5": 0.8402450680732727, + "0.6": 0.8441652854283651, + "0.7": 0.843760629494985, + "0.8": 0.84363720814387, + "0.9": 0.84363720814387 + }, + "matched_cos_abs_band": 0.8438000828027725, + "matched_cos_abs_band_std": 0.001149408075698793, + "unique_frac_abs_per_threshold": { + "0.5": 0.8244900173611112, + "0.6": 0.9948187934027778, + "0.7": 0.9994303385416666, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_abs_band": 0.9985622829861112, + "unique_frac_abs_band_std": 5.873110648473362e-06, + "matched_frac_abs_per_threshold": { + "0.5": 0.7700466579861112, + "0.6": 0.9069281684027778, + "0.7": 0.8368598090277778, + "0.8": 0.717041015625, + "0.9": 0.514865451388889 + }, + "matched_frac_abs_band": 0.743923611111111, + "matched_frac_abs_band_std": 0.0033253011024323635, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 12288 + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/config.json b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9979fab305399fdb0696f029a310a2e2af9a2f59 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 4.5, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 10, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..7cf938d6b64811af3b1eba602dd93007227583bf --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 126.4695044921875, + "l0_per_token_0.001": 126.46924731445313, + "l0_per_token_0.01": 126.46831743164063, + "l0_per_token_0.05": 126.46364501953126, + "l0_per_token_0.1": 126.454286328125, + "l0_per_token_0.5": 111.81361259765625, + "l0_per_token_1.0": 79.31354155273438, + "l1_per_token": 289.3836344397387, + "r2_score": 0.9436933468059865, + "l2_loss": 1.3205280686668774 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/config.json b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..379ab6e036d7af49084db9e8fd480270d463ae8b --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 4.5, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 10, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..32483604c9e95e517d8c0ea49f422b1dc96ae209 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 125.89824155273438, + "l0_per_token_0.001": 125.89810434570313, + "l0_per_token_0.01": 125.89738203125, + "l0_per_token_0.05": 125.89346733398438, + "l0_per_token_0.1": 125.88476430664062, + "l0_per_token_0.5": 111.54088544921875, + "l0_per_token_1.0": 79.13532709960937, + "l1_per_token": 289.51439305719344, + "r2_score": 0.9437663728355269, + "l2_loss": 1.3188154305276674 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/config.json b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b9b845132d9a10f1611ed127e3e28b0febd52a3c --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 4.5, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 10, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..30fba869c5b07010efb9d0f651b472151c91c74e --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 126.127159765625, + "l0_per_token_0.001": 126.12707016601563, + "l0_per_token_0.01": 126.12630732421874, + "l0_per_token_0.05": 126.12196870117188, + "l0_per_token_0.1": 126.11321782226563, + "l0_per_token_0.5": 111.70350498046875, + "l0_per_token_1.0": 79.29182919921875, + "l1_per_token": 289.6085860579297, + "r2_score": 0.9437536036438203, + "l2_loss": 1.3191148991535948 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_32/code_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x16/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..47b2c7fff2bc9d6427b5870e8e4c43a488b0d5c1 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 34.11666833333334, + "recon_r2": 0.8986679004156218, + "l1_per_token": 186.22070988001735 + }, + "seed1": { + "l0": 34.10902166666666, + "recon_r2": 0.8987894297283623, + "l1_per_token": 185.72599150414578 + }, + "seed2": { + "l0": 34.240206666666666, + "recon_r2": 0.8987818586148901, + "l1_per_token": 186.73523437882213 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7809844014446874, + "mcc_abs": 0.780985660585697, + "act_iou": 0.5074479851575096, + "act_iou_n_pairs": 12265, + "input_r2": 0.9863562707505128, + "matched_mcc_signed": { + "0.6": 0.8304022051494929, + "0.7": 0.857498462100996, + "0.8": 0.8958479882608926, + "0.9": 0.9414620631216186 + }, + "matched_mcc_abs": { + "0.6": 0.8308284407037294, + "0.7": 0.8573308349313363, + "0.8": 0.895670010459787, + "0.9": 0.9414558876475153 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7779173925424955, + "mcc_abs": 0.7779183138998308, + "act_iou": 0.5054622423786899, + "act_iou_n_pairs": 12271, + "input_r2": 0.9863696984638582, + "matched_mcc_signed": { + "0.6": 0.8251178698395323, + "0.7": 0.8545442564293723, + "0.8": 0.8941268562305976, + "0.9": 0.9405185795513432 + }, + "matched_mcc_abs": { + "0.6": 0.8252502187072939, + "0.7": 0.8545884774945348, + "0.8": 0.8943297597748052, + "0.9": 0.9404768194422161 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7805839115016303, + "mcc_abs": 0.7805845164009844, + "act_iou": 0.5090701067602436, + "act_iou_n_pairs": 12268, + "input_r2": 0.9864737100906098, + "matched_mcc_signed": { + "0.6": 0.8274534585829221, + "0.7": 0.8567122593646093, + "0.8": 0.8951611853298442, + "0.9": 0.9415611179795759 + }, + "matched_mcc_abs": { + "0.6": 0.8272814411540946, + "0.7": 0.8561547899705673, + "0.8": 0.8952639714169829, + "0.9": 0.9415610600249489 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 34.155298888888886, + "l0_std": 0.0736316231190653, + "recon_r2_mean": 0.8987463962529582, + "recon_r2_std": 6.808471046528173e-05, + "l1_per_token_mean": 186.2273119209951, + "l1_per_token_std": 0.5046538271231552, + "input_r2_mean": 0.9863998931016603, + "input_r2_std": 6.427897606995591e-05, + "mcc_signed_mean": 0.779828568496271, + "mcc_signed_std": 0.0016671962072113684, + "mcc_abs_mean": 0.7798294969621707, + "mcc_abs_std": 0.0016672416404239093, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8276578445239825, + "0.7": 0.8562516592983259, + "0.8": 0.8950453432737783, + "0.9": 0.9411805868841793 + }, + "matched_mcc_signed_band": 0.8800338584950664, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8277867001883726, + "0.7": 0.8560247007988128, + "0.8": 0.8950879138838584, + "0.9": 0.9411645890382268 + }, + "matched_mcc_abs_band": 0.8800159759773176, + "act_iou_mean": 0.5073267780988143, + "act_iou_std": 0.001806983594897617, + "act_iou_n_pairs": 12268.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_32/dict_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x16/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..021f9c94162e2faf43ba9e699e254ae2daa03ab1 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_32/dict_metrics_step_final.json @@ -0,0 +1,550 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 34, + "l0_raw": 34.09826708984375, + "recon_r2": 0.8984899544332474, + "l1_per_token": 186.09178804915646, + "stats_source": "stats", + "n_dead": 32, + "dead_rate": 0.0026041666666666665, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 5875, + "0.6": 123, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 2667, + "0.6": 60, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 3, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.48740267753601074, + "n_dup_abs": { + "0.5": 5894, + "0.6": 193, + "0.7": 12, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 2671, + "0.6": 95, + "0.7": 6, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 3, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_abs": 0.4891930818557739 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 34, + "l0_raw": 34.087190966796875, + "recon_r2": 0.8986077841689825, + "l1_per_token": 185.59309232967692, + "stats_source": "stats", + "n_dead": 38, + "dead_rate": 0.0030924479166666665, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 5935, + "0.6": 127, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 2682, + "0.6": 63, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 3, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.4871804714202881, + "n_dup_abs": { + "0.5": 5942, + "0.6": 195, + "0.7": 16, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 2684, + "0.6": 97, + "0.7": 8, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 3, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.4889828860759735 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 34, + "l0_raw": 34.21214306640625, + "recon_r2": 0.898614796428707, + "l1_per_token": 186.597777168898, + "stats_source": "stats", + "n_dead": 25, + "dead_rate": 0.0020345052083333335, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 5966, + "0.6": 142, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 2699, + "0.6": 70, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 7, + "0.6": 3, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.48869165778160095, + "n_dup_abs": { + "0.5": 5969, + "0.6": 212, + "0.7": 14, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 2699, + "0.6": 105, + "0.7": 7, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 7, + "0.6": 3, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_abs": 0.49046310782432556 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8540685176849365, + "dcs_abs": 0.8541384339332581, + "frac_sign_flipped": 0.00341796875, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12256, + "n_alive_b": 12250, + "matched_cos_signed": { + "0.5": 0.8559465408325195, + "0.6": 0.8553089499473572, + "0.7": 0.8564656376838684, + "0.8": 0.8564656376838684, + "0.9": 0.8564656376838684 + }, + "unique_frac_signed": { + "0.5": 0.7342529296875, + "0.6": 0.9919840494791666, + "0.7": 0.9971516927083334, + "0.8": 0.9971516927083334, + "0.9": 0.9971516927083334 + }, + "dedup_dcs_signed": { + "0.5": 0.6284812552621588, + "0.6": 0.8484528357245532, + "0.7": 0.8540261603629915, + "0.8": 0.8540261603629915, + "0.9": 0.8540261603629915 + }, + "matched_frac_signed": { + "0.5": 0.689208984375, + "0.6": 0.9131673177083334, + "0.7": 0.85888671875, + "0.8": 0.7469889322916666, + "0.9": 0.548828125 + }, + "matched_cos_abs": { + "0.5": 0.8558028340339661, + "0.6": 0.8556124567985535, + "0.7": 0.8565983176231384, + "0.8": 0.8565321564674377, + "0.9": 0.8565024137496948 + }, + "unique_frac_abs": { + "0.5": 0.7334391276041666, + "0.6": 0.9891764322916666, + "0.7": 0.99658203125, + "0.8": 0.9971110026041666, + "0.9": 0.9971516927083334 + }, + "dedup_dcs_abs": { + "0.5": 0.6276792839950455, + "0.6": 0.8463516774403008, + "0.7": 0.8536704913422, + "0.8": 0.8540576372979558, + "0.9": 0.8540628316792815 + }, + "matched_frac_abs": { + "0.5": 0.6878255208333334, + "0.6": 0.9100748697916666, + "0.7": 0.8588053385416666, + "0.8": 0.7470703125, + "0.9": 0.5489908854166666 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8531638979911804, + "dcs_abs": 0.8532828688621521, + "frac_sign_flipped": 0.004069010416666667, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12256, + "n_alive_b": 12263, + "matched_cos_signed": { + "0.5": 0.8537924885749817, + "0.6": 0.8539842367172241, + "0.7": 0.8551971912384033, + "0.8": 0.8551971912384033, + "0.9": 0.8551971912384033 + }, + "unique_frac_signed": { + "0.5": 0.7342122395833334, + "0.6": 0.9921875, + "0.7": 0.9976806640625, + "0.8": 0.9976806640625, + "0.9": 0.9976806640625 + }, + "dedup_dcs_signed": { + "0.5": 0.6268648951760649, + "0.6": 0.8473124848678708, + "0.7": 0.853213701659115, + "0.8": 0.853213701659115, + "0.9": 0.853213701659115 + }, + "matched_frac_signed": { + "0.5": 0.6895345052083334, + "0.6": 0.914306640625, + "0.7": 0.8561197916666666, + "0.8": 0.7405598958333334, + "0.9": 0.5435384114583334 + }, + "matched_cos_abs": { + "0.5": 0.8535830974578857, + "0.6": 0.8543457984924316, + "0.7": 0.855301558971405, + "0.8": 0.8552937507629395, + "0.9": 0.8552937507629395 + }, + "unique_frac_abs": { + "0.5": 0.7334798177083334, + "0.6": 0.9893391927083334, + "0.7": 0.9971516927083334, + "0.8": 0.9976806640625, + "0.9": 0.9976806640625 + }, + "dedup_dcs_abs": { + "0.5": 0.6260859747223246, + "0.6": 0.8452377825742587, + "0.7": 0.8528653973044129, + "0.8": 0.8533100372296758, + "0.9": 0.8533100372296758 + }, + "matched_frac_abs": { + "0.5": 0.6883951822916666, + "0.6": 0.9117838541666666, + "0.7": 0.8553059895833334, + "0.8": 0.7406412760416666, + "0.9": 0.543701171875 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.854972779750824, + "dcs_abs": 0.8550770878791809, + "frac_sign_flipped": 0.00341796875, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12250, + "n_alive_b": 12263, + "matched_cos_signed": { + "0.5": 0.8539811968803406, + "0.6": 0.8557738065719604, + "0.7": 0.8573965430259705, + "0.8": 0.8573965430259705, + "0.9": 0.8573965430259705 + }, + "unique_frac_signed": { + "0.5": 0.7321370442708334, + "0.6": 0.9919026692708334, + "0.7": 0.9974365234375, + "0.8": 0.9974365234375, + "0.9": 0.9974365234375 + }, + "dedup_dcs_signed": { + "0.5": 0.6252312693468411, + "0.6": 0.8488443230307894, + "0.7": 0.8551986270831549, + "0.8": 0.8551986270831549, + "0.9": 0.8551986270831549 + }, + "matched_frac_signed": { + "0.5": 0.686767578125, + "0.6": 0.9159342447916666, + "0.7": 0.8600260416666666, + "0.8": 0.74658203125, + "0.9": 0.5498046875 + }, + "matched_cos_abs": { + "0.5": 0.854003369808197, + "0.6": 0.8561245799064636, + "0.7": 0.8575484752655029, + "0.8": 0.8574988842010498, + "0.9": 0.8574696779251099 + }, + "unique_frac_abs": { + "0.5": 0.7318115234375, + "0.6": 0.9890950520833334, + "0.7": 0.996826171875, + "0.8": 0.9973958333333334, + "0.9": 0.9974365234375 + }, + "dedup_dcs_abs": { + "0.5": 0.6249695070800954, + "0.6": 0.8467885859524056, + "0.7": 0.8548267637961544, + "0.8": 0.8552658141901096, + "0.9": 0.8552715745026944 + }, + "matched_frac_abs": { + "0.5": 0.6865234375, + "0.6": 0.91357421875, + "0.7": 0.8600260416666666, + "0.8": 0.7462565104166666, + "0.9": 0.5498046875 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 32, + "n_dead_std": 6.506407098647712, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": 34, + "l0_std": 0.06916579807077423, + "recon_r2_mean": 0.8985708450103124, + "recon_r2_std": 7.01409797481315e-05, + "l1_per_token_mean": 186.09421918257712, + "l1_per_token_std": 0.5023468317282951, + "babel_mu1_signed_mean": 0.48775826891263324, + "babel_mu1_signed_std": 0.0008159380950787432, + "n_dup_signed_per_threshold": { + "0.5": 5925.333333333333, + "0.6": 130.66666666666666, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 32.666666666666664, + "n_dup_signed_band_std": 2.504163200219453, + "n_dup_clusters_signed_per_threshold": { + "0.5": 2682.6666666666665, + "0.6": 64.33333333333333, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 16.083333333333332, + "n_dup_clusters_signed_band_std": 1.282900359861721, + "largest_dup_signed_per_threshold": { + "0.5": 6.333333333333333, + "0.6": 3.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.5, + "largest_dup_signed_band_std": 0.0, + "dcs_signed_mean": 0.854068398475647, + "dcs_signed_std": 0.0009044408857138932, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.626859139928355, + "0.6": 0.8482032145410711, + "0.7": 0.8541461630350872, + "0.8": 0.8541461630350872, + "0.9": 0.8541461630350872 + }, + "dedup_dcs_signed_band": 0.8526604259115832, + "dedup_dcs_signed_band_std": 0.0009361319306183756, + "matched_cos_signed_per_threshold": { + "0.5": 0.8545734087626139, + "0.6": 0.8550223310788473, + "0.7": 0.8563531239827474, + "0.8": 0.8563531239827474, + "0.9": 0.8563531239827474 + }, + "matched_cos_signed_band": 0.8560204257567724, + "matched_cos_signed_band_std": 0.0010571259962131921, + "unique_frac_signed_per_threshold": { + "0.5": 0.7335340711805557, + "0.6": 0.9920247395833334, + "0.7": 0.9974229600694445, + "0.8": 0.9974229600694445, + "0.9": 0.9974229600694445 + }, + "unique_frac_signed_band": 0.9960734049479167, + "unique_frac_signed_band_std": 0.00022448808279124232, + "matched_frac_signed_per_threshold": { + "0.5": 0.6885036892361112, + "0.6": 0.9144694010416666, + "0.7": 0.8583441840277777, + "0.8": 0.7447102864583334, + "0.9": 0.5473904079861112 + }, + "matched_frac_signed_band": 0.7662285698784722, + "matched_frac_signed_band_std": 0.0023179375372052525, + "babel_mu1_abs_mean": 0.48954635858535767, + "babel_mu1_abs_std": 0.0008008541951083467, + "n_dup_abs_per_threshold": { + "0.5": 5935.0, + "0.6": 200.0, + "0.7": 14.0, + "0.8": 0.6666666666666666, + "0.9": 0.0 + }, + "n_dup_abs_band": 53.666666666666664, + "n_dup_abs_band_std": 2.6496855159307744, + "n_dup_clusters_abs_per_threshold": { + "0.5": 2684.6666666666665, + "0.6": 99.0, + "0.7": 7.0, + "0.8": 0.3333333333333333, + "0.9": 0.0 + }, + "n_dup_clusters_abs_band": 26.583333333333332, + "n_dup_clusters_abs_band_std": 1.3768926368215255, + "largest_dup_abs_per_threshold": { + "0.5": 6.333333333333333, + "0.6": 3.0, + "0.7": 2.0, + "0.8": 1.3333333333333333, + "0.9": 1.0 + }, + "largest_dup_abs_band": 1.8333333333333333, + "largest_dup_abs_band_std": 0.14433756729740646, + "dcs_abs_mean": 0.8541661302248637, + "dcs_abs_std": 0.0008974300996141616, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6262449219324885, + "0.6": 0.8461260153223217, + "0.7": 0.8537875508142557, + "0.8": 0.8542111629059138, + "0.9": 0.8542148144705507 + }, + "dedup_dcs_abs_band": 0.8520848858782605, + "dedup_dcs_abs_band_std": 0.0009296634948064181, + "matched_cos_abs_per_threshold": { + "0.5": 0.8544631004333496, + "0.6": 0.8553609450658163, + "0.7": 0.8564827839533488, + "0.8": 0.856441597143809, + "0.9": 0.856421947479248 + }, + "matched_cos_abs_band": 0.8561768184105555, + "matched_cos_abs_band_std": 0.0010572823834023232, + "unique_frac_abs_per_threshold": { + "0.5": 0.73291015625, + "0.6": 0.9892035590277778, + "0.7": 0.9968532986111112, + "0.8": 0.9973958333333334, + "0.9": 0.9974229600694445 + }, + "unique_frac_abs_band": 0.9952189127604167, + "unique_frac_abs_band_std": 0.00023040266220924184, + "matched_frac_abs_per_threshold": { + "0.5": 0.6875813802083334, + "0.6": 0.9118109809027777, + "0.7": 0.8580457899305555, + "0.8": 0.744656032986111, + "0.9": 0.5474989149305555 + }, + "matched_frac_abs_band": 0.7655029296874999, + "matched_frac_abs_band_std": 0.0023652818407118256, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 12288 + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/config.json b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd2932a390519c73715b7a1823c3bc7358fba4b9 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 12.5, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 30, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..4161a223a20d416579362578d108fa7a8f647603 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 34.09826708984375, + "l0_per_token_0.001": 34.098253857421874, + "l0_per_token_0.01": 34.09807836914062, + "l0_per_token_0.05": 34.09750639648438, + "l0_per_token_0.1": 34.096228662109375, + "l0_per_token_0.5": 34.0711404296875, + "l0_per_token_1.0": 34.01213842773438, + "l1_per_token": 186.09178804915646, + "r2_score": 0.8984899544332474, + "l2_loss": 2.3806576455658055 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/config.json b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbcb0ffdab198f7fd6b2f812895a35e8ff8dbcb4 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 12.5, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 30, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..2a1c1e0f8e0aba61425d3bc427314aeed8e543c9 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 34.087190966796875, + "l0_per_token_0.001": 34.0871806640625, + "l0_per_token_0.01": 34.087062158203125, + "l0_per_token_0.05": 34.086601708984375, + "l0_per_token_0.1": 34.08566435546875, + "l0_per_token_0.5": 34.06172890625, + "l0_per_token_1.0": 34.00160493164063, + "l1_per_token": 185.59309232967692, + "r2_score": 0.8986077841689825, + "l2_loss": 2.377894251463415 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/config.json b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..eaac861df067c6995f866db495707dbd64e270d7 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 12.5, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 30, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..5d7ff97ab28603a6d6df2994450f055773c423fd --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 34.21214306640625, + "l0_per_token_0.001": 34.21213720703125, + "l0_per_token_0.01": 34.21210283203125, + "l0_per_token_0.05": 34.21181171875, + "l0_per_token_0.1": 34.21116733398438, + "l0_per_token_0.5": 34.19097841796875, + "l0_per_token_1.0": 34.13543442382812, + "l1_per_token": 186.597777168898, + "r2_score": 0.898614796428707, + "l2_loss": 2.3777297969049256 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_64/code_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x16/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..5a5b30e22aea9c239f08b710584d42b656edbca3 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 68.33619833333333, + "recon_r2": 0.9322517340770707, + "l1_per_token": 255.8558036472112 + }, + "seed1": { + "l0": 68.25313666666666, + "recon_r2": 0.9323407862864682, + "l1_per_token": 255.34123201401795 + }, + "seed2": { + "l0": 68.50301666666667, + "recon_r2": 0.9322924700450567, + "l1_per_token": 256.96076816634786 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7768021305413016, + "mcc_abs": 0.7768030977677247, + "act_iou": 0.46257098043952416, + "act_iou_n_pairs": 12288, + "input_r2": 0.9875357167160536, + "matched_mcc_signed": { + "0.6": 0.8336217505790626, + "0.7": 0.8617261810456283, + "0.8": 0.8989884177254652, + "0.9": 0.9414152728089523 + }, + "matched_mcc_abs": { + "0.6": 0.8338279584249467, + "0.7": 0.8612219114060355, + "0.8": 0.898788023707797, + "0.9": 0.9413730426214577 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.775137631695522, + "mcc_abs": 0.7751397889782531, + "act_iou": 0.46259933192133634, + "act_iou_n_pairs": 12288, + "input_r2": 0.9876135734799633, + "matched_mcc_signed": { + "0.6": 0.8327112898532983, + "0.7": 0.8610406097487728, + "0.8": 0.8985328214583472, + "0.9": 0.9411809905755064 + }, + "matched_mcc_abs": { + "0.6": 0.8327528887150836, + "0.7": 0.8610792045865284, + "0.8": 0.8986460702756782, + "0.9": 0.9411762675459137 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.775594728791976, + "mcc_abs": 0.7755953713512026, + "act_iou": 0.46374883008495243, + "act_iou_n_pairs": 12288, + "input_r2": 0.987641363010334, + "matched_mcc_signed": { + "0.6": 0.833073454668621, + "0.7": 0.8626854635967414, + "0.8": 0.9002780058304956, + "0.9": 0.9413255147035597 + }, + "matched_mcc_abs": { + "0.6": 0.8339831061502786, + "0.7": 0.8627077226289882, + "0.8": 0.8999366437006253, + "0.9": 0.9412923738555892 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 68.36411722222222, + "l0_std": 0.12725801297990336, + "recon_r2_mean": 0.9322949968028652, + "recon_r2_std": 4.457984273649158e-05, + "l1_per_token_mean": 256.052601275859, + "l1_per_token_std": 0.8275091039722443, + "input_r2_mean": 0.9875968844021169, + "input_r2_std": 5.4764759319457435e-05, + "mcc_signed_mean": 0.7758448303429333, + "mcc_signed_std": 0.0008599722037623513, + "mcc_abs_mean": 0.7758460860323936, + "mcc_abs_std": 0.0008595303489737128, + "matched_mcc_signed_per_threshold": { + "0.6": 0.833135498366994, + "0.7": 0.8618174181303808, + "0.8": 0.8992664150047694, + "0.9": 0.9413072593626728 + }, + "matched_mcc_signed_band": 0.8838816477162043, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8335213177634363, + "0.7": 0.8616696128738507, + "0.8": 0.8991235792280335, + "0.9": 0.9412805613409869 + }, + "matched_mcc_abs_band": 0.8838987678015768, + "act_iou_mean": 0.4629730474819376, + "act_iou_std": 0.0006719969769169633, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_64/dict_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x16/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..4c82202d3dbba401c280c645a52b1ae20a6267a5 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_64/dict_metrics_step_final.json @@ -0,0 +1,550 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 68, + "l0_raw": 68.38207143554688, + "recon_r2": 0.9321300780911635, + "l1_per_token": 255.81543037355635, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 14, + "dense_rate": 0.0011393229166666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3315, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1541, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 5, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.4383678436279297, + "n_dup_abs": { + "0.5": 3347, + "0.6": 110, + "0.7": 30, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 1557, + "0.6": 55, + "0.7": 15, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 5, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.4412466585636139 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 68, + "l0_raw": 68.27429501953125, + "recon_r2": 0.9322215810946727, + "l1_per_token": 255.2836143987286, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 16, + "dense_rate": 0.0013020833333333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3240, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1500, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 7, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.4383070468902588, + "n_dup_abs": { + "0.5": 3280, + "0.6": 98, + "0.7": 18, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 1518, + "0.6": 49, + "0.7": 9, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 7, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_abs": 0.44103512167930603 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 69, + "l0_raw": 68.547587109375, + "recon_r2": 0.9321723959031226, + "l1_per_token": 256.91514224984536, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 13, + "dense_rate": 0.0010579427083333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 3275, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1520, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.438535213470459, + "n_dup_abs": { + "0.5": 3318, + "0.6": 98, + "0.7": 16, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 1541, + "0.6": 49, + "0.7": 8, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_abs": 0.4413144290447235 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.842194139957428, + "dcs_abs": 0.8423486351966858, + "frac_sign_flipped": 0.0035807291666666665, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8376299738883972, + "0.6": 0.8421900868415833, + "0.7": 0.842194139957428, + "0.8": 0.842194139957428, + "0.9": 0.842194139957428 + }, + "unique_frac_signed": { + "0.5": 0.8570149739583334, + "0.6": 0.9999186197916666, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7178614302586842, + "0.6": 0.8421215492368598, + "0.7": 0.842194139957428, + "0.8": 0.842194139957428, + "0.9": 0.842194139957428 + }, + "matched_frac_signed": { + "0.5": 0.7935384114583334, + "0.6": 0.903076171875, + "0.7": 0.835205078125, + "0.8": 0.7178548177083334, + "0.9": 0.5221354166666666 + }, + "matched_cos_abs": { + "0.5": 0.8379883766174316, + "0.6": 0.8430367708206177, + "0.7": 0.8426423668861389, + "0.8": 0.8423916697502136, + "0.9": 0.8423486351966858 + }, + "unique_frac_abs": { + "0.5": 0.85546875, + "0.6": 0.9957682291666666, + "0.7": 0.9990234375, + "0.8": 0.9999593098958334, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.7168728690594435, + "0.6": 0.8394692324024314, + "0.7": 0.8418194739497267, + "0.8": 0.8423573927454224, + "0.9": 0.8423486351966858 + }, + "matched_frac_abs": { + "0.5": 0.7913411458333334, + "0.6": 0.8990885416666666, + "0.7": 0.83447265625, + "0.8": 0.71826171875, + "0.9": 0.5224609375 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8415024280548096, + "dcs_abs": 0.8416246771812439, + "frac_sign_flipped": 0.0033365885416666665, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8373281359672546, + "0.6": 0.8414965867996216, + "0.7": 0.8415024280548096, + "0.8": 0.8415024280548096, + "0.9": 0.8415024280548096 + }, + "unique_frac_signed": { + "0.5": 0.8564046223958334, + "0.6": 0.9999186197916666, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7170916861044437, + "0.6": 0.841428105632076, + "0.7": 0.8415024280548096, + "0.8": 0.8415024280548096, + "0.9": 0.8415024280548096 + }, + "matched_frac_signed": { + "0.5": 0.7937825520833334, + "0.6": 0.9007161458333334, + "0.7": 0.8323567708333334, + "0.8": 0.7145182291666666, + "0.9": 0.52197265625 + }, + "matched_cos_abs": { + "0.5": 0.8376181721687317, + "0.6": 0.8422266244888306, + "0.7": 0.8418663144111633, + "0.8": 0.8416663408279419, + "0.9": 0.8416246771812439 + }, + "unique_frac_abs": { + "0.5": 0.8548583984375, + "0.6": 0.9957682291666666, + "0.7": 0.9990641276041666, + "0.8": 0.9999593098958334, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.7160449291623081, + "0.6": 0.8386625144242619, + "0.7": 0.8410784349665239, + "0.8": 0.8416320933368601, + "0.9": 0.8416246771812439 + }, + "matched_frac_abs": { + "0.5": 0.79296875, + "0.6": 0.8961588541666666, + "0.7": 0.8316243489583334, + "0.8": 0.714599609375, + "0.9": 0.5220540364583334 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8420475125312805, + "dcs_abs": 0.8422133326530457, + "frac_sign_flipped": 0.004150390625, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8369420766830444, + "0.6": 0.8420504331588745, + "0.7": 0.8420475125312805, + "0.8": 0.8420475125312805, + "0.9": 0.8420475125312805 + }, + "unique_frac_signed": { + "0.5": 0.8577880859375, + "0.6": 0.9999186197916666, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.717918941998505, + "0.6": 0.8419819069191968, + "0.7": 0.8420475125312805, + "0.8": 0.8420475125312805, + "0.9": 0.8420475125312805 + }, + "matched_frac_signed": { + "0.5": 0.7963053385416666, + "0.6": 0.9016927083333334, + "0.7": 0.8310546875, + "0.8": 0.7145182291666666, + "0.9": 0.526611328125 + }, + "matched_cos_abs": { + "0.5": 0.8375152945518494, + "0.6": 0.8427015542984009, + "0.7": 0.8422761559486389, + "0.8": 0.8422133326530457, + "0.9": 0.8422133326530457 + }, + "unique_frac_abs": { + "0.5": 0.8559977213541666, + "0.6": 0.9960123697916666, + "0.7": 0.9993082682291666, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.7169111837356468, + "0.6": 0.839341172123871, + "0.7": 0.8416935267717538, + "0.8": 0.8422133326530457, + "0.9": 0.8422133326530457 + }, + "matched_frac_abs": { + "0.5": 0.794921875, + "0.6": 0.8975423177083334, + "0.7": 0.8302408854166666, + "0.8": 0.7154134114583334, + "0.9": 0.5266927083333334 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 14, + "n_dense_std": 1.5275252316519468, + "l0_mean": 68, + "l0_std": 0.13766530314758454, + "recon_r2_mean": 0.932174685029653, + "recon_r2_std": 3.464368346113687e-05, + "l1_per_token_mean": 256.00472900737674, + "l1_per_token_std": 0.8320763648757366, + "babel_mu1_signed_mean": 0.4384033679962158, + "babel_mu1_signed_std": 0.00011815871798838899, + "n_dup_signed_per_threshold": { + "0.5": 3276.6666666666665, + "0.6": 2.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.5, + "n_dup_signed_band_std": 0.0, + "n_dup_clusters_signed_per_threshold": { + "0.5": 1520.3333333333333, + "0.6": 1.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.25, + "n_dup_clusters_signed_band_std": 0.0, + "largest_dup_signed_per_threshold": { + "0.5": 6.0, + "0.6": 2.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.25, + "largest_dup_signed_band_std": 0.0, + "dcs_signed_mean": 0.841914693514506, + "dcs_signed_std": 0.00036448183420947374, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7176240194538778, + "0.6": 0.8418438539293774, + "0.7": 0.841914693514506, + "0.8": 0.841914693514506, + "0.9": 0.841914693514506 + }, + "dedup_dcs_signed_band": 0.8418969836182238, + "dedup_dcs_signed_band_std": 0.00036504624021942595, + "matched_cos_signed_per_threshold": { + "0.5": 0.8373000621795654, + "0.6": 0.8419123689333597, + "0.7": 0.841914693514506, + "0.8": 0.841914693514506, + "0.9": 0.841914693514506 + }, + "matched_cos_signed_band": 0.8419141123692194, + "matched_cos_signed_band_std": 0.00036505370224472965, + "unique_frac_signed_per_threshold": { + "0.5": 0.8570692274305557, + "0.6": 0.9999186197916666, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9999796549479166, + "unique_frac_signed_band_std": 0.0, + "matched_frac_signed_per_threshold": { + "0.5": 0.7945421006944445, + "0.6": 0.901828342013889, + "0.7": 0.8328721788194445, + "0.8": 0.7156304253472222, + "0.9": 0.5235731336805555 + }, + "matched_frac_signed_band": 0.7434760199652778, + "matched_frac_signed_band_std": 0.0010884761313991254, + "babel_mu1_abs_mean": 0.4411987364292145, + "babel_mu1_abs_std": 0.00014568990808618186, + "n_dup_abs_per_threshold": { + "0.5": 3315.0, + "0.6": 102.0, + "0.7": 21.333333333333332, + "0.8": 0.6666666666666666, + "0.9": 0.0 + }, + "n_dup_abs_band": 31.0, + "n_dup_abs_band_std": 3.905124837953327, + "n_dup_clusters_abs_per_threshold": { + "0.5": 1538.6666666666667, + "0.6": 51.0, + "0.7": 10.666666666666666, + "0.8": 0.3333333333333333, + "0.9": 0.0 + }, + "n_dup_clusters_abs_band": 15.5, + "n_dup_clusters_abs_band_std": 1.9525624189766635, + "largest_dup_abs_per_threshold": { + "0.5": 6.0, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 1.3333333333333333, + "0.9": 1.0 + }, + "largest_dup_abs_band": 1.5833333333333333, + "largest_dup_abs_band_std": 0.14433756729740646, + "dcs_abs_mean": 0.8420622150103251, + "dcs_abs_std": 0.00038491065000425916, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.7166096606524661, + "0.6": 0.8391576396501881, + "0.7": 0.8415304785626682, + "0.8": 0.8420676062451093, + "0.9": 0.8420622150103251 + }, + "dedup_dcs_abs_band": 0.8412044848670727, + "dedup_dcs_abs_band_std": 0.0003996889687014163, + "matched_cos_abs_per_threshold": { + "0.5": 0.8377072811126709, + "0.6": 0.8426549832026163, + "0.7": 0.8422616124153137, + "0.8": 0.8420904477437338, + "0.9": 0.8420622150103251 + }, + "matched_cos_abs_band": 0.8422673145929972, + "matched_cos_abs_band_std": 0.00038631035693890634, + "unique_frac_abs_per_threshold": { + "0.5": 0.8554416232638888, + "0.6": 0.995849609375, + "0.7": 0.9991319444444443, + "0.8": 0.999972873263889, + "0.9": 1.0 + }, + "unique_frac_abs_band": 0.9987386067708333, + "unique_frac_abs_band_std": 7.944996822004592e-05, + "matched_frac_abs_per_threshold": { + "0.5": 0.7930772569444445, + "0.6": 0.8975965711805555, + "0.7": 0.8321126302083334, + "0.8": 0.7160915798611112, + "0.9": 0.5237358940972223 + }, + "matched_frac_abs_band": 0.7423841688368056, + "matched_frac_abs_band_std": 0.0012332413623357275, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 12288 + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/config.json b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9a58a940d902c932c404461f38b71bdbdb6ba5c8 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 6.5, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 18, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..cf9d6f163219e4ed60c7a09a7ec23983c04595f4 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 68.38207143554688, + "l0_per_token_0.001": 68.3819552734375, + "l0_per_token_0.01": 68.38150571289063, + "l0_per_token_0.05": 68.37984819335938, + "l0_per_token_0.1": 68.37665571289062, + "l0_per_token_0.5": 68.29212431640624, + "l0_per_token_1.0": 63.50503798828125, + "l1_per_token": 255.81543037355635, + "r2_score": 0.9321300780911635, + "l2_loss": 1.5917148652048885 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/config.json b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2208d1705a82d50539e6627d06690487465623b5 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 6.5, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 18, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..dc9b551a8f45cd5cdf8a90ffa4ce44d1a82cf2df --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 68.27429501953125, + "l0_per_token_0.001": 68.27426918945312, + "l0_per_token_0.01": 68.27396088867188, + "l0_per_token_0.05": 68.27259619140625, + "l0_per_token_0.1": 68.269659375, + "l0_per_token_0.5": 68.191526171875, + "l0_per_token_1.0": 63.4634873046875, + "l1_per_token": 255.2836143987286, + "r2_score": 0.9322215810946727, + "l2_loss": 1.5895688970528683 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/config.json b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..11d587dc3828589a0c7fd39979c907521c8c60ba --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 6.5, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 18, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..7b4b84a820eee15efac855c0c9f6e299565f2541 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 68.547587109375, + "l0_per_token_0.001": 68.5475615234375, + "l0_per_token_0.01": 68.54738310546875, + "l0_per_token_0.05": 68.54639868164062, + "l0_per_token_0.1": 68.54444653320313, + "l0_per_token_0.5": 68.47152568359375, + "l0_per_token_1.0": 63.59962797851563, + "l1_per_token": 256.91514224984536, + "r2_score": 0.9321723959031226, + "l2_loss": 1.5907224095122363 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_128/code_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x4/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..a41eacd40eab0fa97bf96c648aac5082c2cd2ed4 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 127.844745, + "recon_r2": 0.9415145041736372, + "l1_per_token": 393.4831958380243 + }, + "seed1": { + "l0": 127.82662833333333, + "recon_r2": 0.9413935058689876, + "l1_per_token": 392.6340377780998 + }, + "seed2": { + "l0": 127.94816, + "recon_r2": 0.9412829085142881, + "l1_per_token": 392.29133534548754 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.8384991348521575, + "mcc_abs": 0.8385107009141827, + "act_iou": 0.5043471564033918, + "act_iou_n_pairs": 3072, + "input_r2": 0.9880527451632783, + "matched_mcc_signed": { + "0.6": 0.8957322305283675, + "0.7": 0.905955677007102, + "0.8": 0.9195577963035144, + "0.9": 0.9397493154646301 + }, + "matched_mcc_abs": { + "0.6": 0.8954973345640642, + "0.7": 0.904681714997588, + "0.8": 0.9186516445179674, + "0.9": 0.9396600462816262 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.8343279700795557, + "mcc_abs": 0.834332741334677, + "act_iou": 0.49883932105666623, + "act_iou_n_pairs": 3072, + "input_r2": 0.9878843753705879, + "matched_mcc_signed": { + "0.6": 0.8940291203862878, + "0.7": 0.9052258916690518, + "0.8": 0.9194332997748622, + "0.9": 0.9378714317629683 + }, + "matched_mcc_abs": { + "0.6": 0.892954798378118, + "0.7": 0.902677393241714, + "0.8": 0.9187113669722669, + "0.9": 0.937871431762968 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.8358634571464081, + "mcc_abs": 0.8358725738918551, + "act_iou": 0.4990281240433236, + "act_iou_n_pairs": 3072, + "input_r2": 0.9878902078566736, + "matched_mcc_signed": { + "0.6": 0.8962142435565161, + "0.7": 0.9058219536751124, + "0.8": 0.9194503320038773, + "0.9": 0.9381452531661661 + }, + "matched_mcc_abs": { + "0.6": 0.8950302618959097, + "0.7": 0.9037448659876602, + "0.8": 0.9182291337122168, + "0.9": 0.9375700971776684 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 127.8731777777778, + "l0_std": 0.06556526244754074, + "recon_r2_mean": 0.9413969728523043, + "recon_r2_std": 0.00011583674864796442, + "l1_per_token_mean": 392.8028563205373, + "l1_per_token_std": 0.6136021786308381, + "input_r2_mean": 0.9879424427968466, + "input_r2_std": 9.556915560898149e-05, + "mcc_signed_mean": 0.8362301873593738, + "mcc_signed_std": 0.0021096260751315627, + "mcc_abs_mean": 0.8362386720469049, + "mcc_abs_std": 0.002112902614022678, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8953251981570571, + "0.7": 0.9056678407837554, + "0.8": 0.919480476027418, + "0.9": 0.9385886667979215 + }, + "matched_mcc_signed_band": 0.914765545441538, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8944941316126972, + "0.7": 0.9037013247423208, + "0.8": 0.9185307150674836, + "0.9": 0.9383671917407543 + }, + "matched_mcc_abs_band": 0.913773340790814, + "act_iou_mean": 0.5007382005011273, + "act_iou_std": 0.0031268728260532867, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_128/dict_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x4/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..baff3b425888f1e3e8f907d4f7861aa380ff4256 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_128/dict_metrics_step_final.json @@ -0,0 +1,550 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 126, + "l0_raw": 125.704651171875, + "recon_r2": 0.9411897999822579, + "l1_per_token": 391.79768565147, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 51, + "dense_rate": 0.0166015625, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 229, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 109, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.33925139904022217, + "n_dup_abs": { + "0.5": 305, + "0.6": 88, + "0.7": 54, + "0.8": 10, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 147, + "0.6": 44, + "0.7": 27, + "0.8": 5, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.35641753673553467 + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 126, + "l0_raw": 125.63611196289062, + "recon_r2": 0.9410739181284478, + "l1_per_token": 390.88694813323394, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 53, + "dense_rate": 0.017252604166666668, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 222, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 105, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 4, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.3414287567138672, + "n_dup_abs": { + "0.5": 296, + "0.6": 80, + "0.7": 46, + "0.8": 10, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 143, + "0.6": 40, + "0.7": 23, + "0.8": 5, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.35803985595703125 + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 126, + "l0_raw": 125.77726040039063, + "recon_r2": 0.9409605591020928, + "l1_per_token": 390.5783639531069, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 55, + "dense_rate": 0.017903645833333332, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 212, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 101, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 4, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.3400411605834961, + "n_dup_abs": { + "0.5": 272, + "0.6": 78, + "0.7": 44, + "0.8": 6, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 131, + "0.6": 39, + "0.7": 22, + "0.8": 3, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.3550417423248291 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8878882527351379, + "dcs_abs": 0.8885080218315125, + "frac_sign_flipped": 0.012044270833333334, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8854705691337585, + "0.6": 0.8878882527351379, + "0.7": 0.8878882527351379, + "0.8": 0.8878882527351379, + "0.9": 0.8878882527351379 + }, + "unique_frac_signed": { + "0.5": 0.96142578125, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.851314233703306, + "0.6": 0.8878882527351379, + "0.7": 0.8878882527351379, + "0.8": 0.8878882527351379, + "0.9": 0.8878882527351379 + }, + "matched_frac_signed": { + "0.5": 0.890625, + "0.6": 0.9153645833333334, + "0.7": 0.8896484375, + "0.8": 0.8453776041666666, + "0.9": 0.7454427083333334 + }, + "matched_cos_abs": { + "0.5": 0.8882692456245422, + "0.6": 0.8909647464752197, + "0.7": 0.8901378512382507, + "0.8": 0.8887239098548889, + "0.9": 0.8885080218315125 + }, + "unique_frac_abs": { + "0.5": 0.9493815104166666, + "0.6": 0.986328125, + "0.7": 0.9918619791666666, + "0.8": 0.9983723958333334, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.8433063980677009, + "0.6": 0.8787835878320038, + "0.7": 0.8828938908603352, + "0.8": 0.8872774191161928, + "0.9": 0.8885080218315125 + }, + "matched_frac_abs": { + "0.5": 0.8802083333333334, + "0.6": 0.904296875, + "0.7": 0.8841145833333334, + "0.8": 0.845703125, + "0.9": 0.74609375 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8845382332801819, + "dcs_abs": 0.8851227760314941, + "frac_sign_flipped": 0.014322916666666666, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8836352229118347, + "0.6": 0.8845381736755371, + "0.7": 0.8845381736755371, + "0.8": 0.8845381736755371, + "0.9": 0.8845381736755371 + }, + "unique_frac_signed": { + "0.5": 0.96240234375, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.8504126095504034, + "0.6": 0.8845381736755371, + "0.7": 0.8845381736755371, + "0.8": 0.8845381736755371, + "0.9": 0.8845381736755371 + }, + "matched_frac_signed": { + "0.5": 0.8922526041666666, + "0.6": 0.908203125, + "0.7": 0.8824869791666666, + "0.8": 0.8382161458333334, + "0.9": 0.74609375 + }, + "matched_cos_abs": { + "0.5": 0.8872216939926147, + "0.6": 0.8873640298843384, + "0.7": 0.8865522742271423, + "0.8": 0.8858667016029358, + "0.9": 0.8851227760314941 + }, + "unique_frac_abs": { + "0.5": 0.9513346354166666, + "0.6": 0.9864908854166666, + "0.7": 0.9920247395833334, + "0.8": 0.9986979166666666, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.8440447267882215, + "0.6": 0.8753765275275024, + "0.7": 0.8794817889671929, + "0.8": 0.8847132293352236, + "0.9": 0.8851227760314941 + }, + "matched_frac_abs": { + "0.5": 0.8831380208333334, + "0.6": 0.8955078125, + "0.7": 0.8779296875, + "0.8": 0.8382161458333334, + "0.9": 0.74609375 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8841626644134521, + "dcs_abs": 0.884857714176178, + "frac_sign_flipped": 0.012369791666666666, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8812102675437927, + "0.6": 0.8841626644134521, + "0.7": 0.8841626644134521, + "0.8": 0.8841626644134521, + "0.9": 0.8841626644134521 + }, + "unique_frac_signed": { + "0.5": 0.962890625, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.8485091052716598, + "0.6": 0.8841626644134521, + "0.7": 0.8841626644134521, + "0.8": 0.8841626644134521, + "0.9": 0.8841626644134521 + }, + "matched_frac_signed": { + "0.5": 0.8870442708333334, + "0.6": 0.9065755208333334, + "0.7": 0.8834635416666666, + "0.8": 0.8408203125, + "0.9": 0.7425130208333334 + }, + "matched_cos_abs": { + "0.5": 0.8841249346733093, + "0.6": 0.8862740993499756, + "0.7": 0.8853506445884705, + "0.8": 0.8853362202644348, + "0.9": 0.884857714176178 + }, + "unique_frac_abs": { + "0.5": 0.9521484375, + "0.6": 0.9871419270833334, + "0.7": 0.99267578125, + "0.8": 0.9986979166666666, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.841818175103981, + "0.6": 0.8748783223563805, + "0.7": 0.878866142797051, + "0.8": 0.8841834387276322, + "0.9": 0.884857714176178 + }, + "matched_frac_abs": { + "0.5": 0.8772786458333334, + "0.6": 0.896484375, + "0.7": 0.8792317708333334, + "0.8": 0.841796875, + "0.9": 0.7425130208333334 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 53, + "n_dense_std": 2.0, + "l0_mean": 126, + "l0_std": 0.0705839980006476, + "recon_r2_mean": 0.9410747590709327, + "recon_r2_std": 0.00011462275373120408, + "l1_per_token_mean": 391.0876659126036, + "l1_per_token_std": 0.6339574680063991, + "babel_mu1_signed_mean": 0.34024043877919513, + "babel_mu1_signed_std": 0.0011022728605957857, + "n_dup_signed_per_threshold": { + "0.5": 221.0, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.0, + "n_dup_signed_band_std": 0.0, + "n_dup_clusters_signed_per_threshold": { + "0.5": 105.0, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.0, + "n_dup_clusters_signed_band_std": 0.0, + "largest_dup_signed_per_threshold": { + "0.5": 3.6666666666666665, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.0, + "largest_dup_signed_band_std": 0.0, + "dcs_signed_mean": 0.8855297168095907, + "dcs_signed_std": 0.002051165955618181, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.8500786495084564, + "0.6": 0.8855296969413757, + "0.7": 0.8855296969413757, + "0.8": 0.8855296969413757, + "0.9": 0.8855296969413757 + }, + "dedup_dcs_signed_band": 0.8855296969413757, + "dedup_dcs_signed_band_std": 0.002051180361571077, + "matched_cos_signed_per_threshold": { + "0.5": 0.8834386865297953, + "0.6": 0.8855296969413757, + "0.7": 0.8855296969413757, + "0.8": 0.8855296969413757, + "0.9": 0.8855296969413757 + }, + "matched_cos_signed_band": 0.8855296969413757, + "matched_cos_signed_band_std": 0.002051180361571077, + "unique_frac_signed_per_threshold": { + "0.5": 0.9622395833333334, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 1.0, + "unique_frac_signed_band_std": 0.0, + "matched_frac_signed_per_threshold": { + "0.5": 0.8899739583333334, + "0.6": 0.9100477430555557, + "0.7": 0.8851996527777777, + "0.8": 0.8414713541666666, + "0.9": 0.7446831597222223 + }, + "matched_frac_signed_band": 0.8453504774305556, + "matched_frac_signed_band_std": 0.003131111667736198, + "babel_mu1_abs_mean": 0.356499711672465, + "babel_mu1_abs_std": 0.0015007451076361096, + "n_dup_abs_per_threshold": { + "0.5": 291.0, + "0.6": 82.0, + "0.7": 48.0, + "0.8": 8.666666666666666, + "0.9": 0.0 + }, + "n_dup_abs_band": 34.666666666666664, + "n_dup_abs_band_std": 3.055050463303893, + "n_dup_clusters_abs_per_threshold": { + "0.5": 140.33333333333334, + "0.6": 41.0, + "0.7": 24.0, + "0.8": 4.333333333333333, + "0.9": 0.0 + }, + "n_dup_clusters_abs_band": 17.333333333333332, + "n_dup_clusters_abs_band_std": 1.5275252316519465, + "largest_dup_abs_per_threshold": { + "0.5": 3.6666666666666665, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 1.0 + }, + "largest_dup_abs_band": 1.75, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.8861628373463949, + "dcs_abs_std": 0.0020353088583094994, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.8430564333199678, + "0.6": 0.8763461459052956, + "0.7": 0.8804139408748597, + "0.8": 0.8853913623930162, + "0.9": 0.8861628373463949 + }, + "dedup_dcs_abs_band": 0.8820785716298917, + "dedup_dcs_abs_band_std": 0.001995054879145602, + "matched_cos_abs_per_threshold": { + "0.5": 0.8865386247634888, + "0.6": 0.8882009585698446, + "0.7": 0.8873469233512878, + "0.8": 0.8866422772407532, + "0.9": 0.8861628373463949 + }, + "matched_cos_abs_band": 0.8870882491270702, + "matched_cos_abs_band_std": 0.0021952477216257327, + "unique_frac_abs_per_threshold": { + "0.5": 0.950954861111111, + "0.6": 0.9866536458333334, + "0.7": 0.9921875, + "0.8": 0.9985894097222222, + "0.9": 1.0 + }, + "unique_frac_abs_band": 0.994357638888889, + "unique_frac_abs_band_std": 0.0002486206431725214, + "matched_frac_abs_per_threshold": { + "0.5": 0.8802083333333334, + "0.6": 0.8987630208333334, + "0.7": 0.8804253472222223, + "0.8": 0.8419053819444445, + "0.9": 0.7449001736111112 + }, + "matched_frac_abs_band": 0.8414984809027779, + "matched_frac_abs_band_std": 0.003090662747715338, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 3072 + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/config.json b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b74d3cf848053f61a379daf972db26f20f15589a --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 2.95, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 5, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..c3283fda5f3483372839c6c6ea86eb1b4e4bcb4c --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 125.704651171875, + "l0_per_token_0.001": 125.7046107421875, + "l0_per_token_0.01": 125.7044861328125, + "l0_per_token_0.05": 125.70372827148438, + "l0_per_token_0.1": 125.70257119140625, + "l0_per_token_0.5": 125.60522626953124, + "l0_per_token_1.0": 94.27361411132813, + "l1_per_token": 391.79768565147, + "r2_score": 0.9411897999822579, + "l2_loss": 1.3792423353551146 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/config.json b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..278ec84c309df5ff6c339ff5a107094f672bec7d --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 2.95, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 5, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..1890a8cbe0550582a3fb438acd09b4958330d86a --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 125.63611196289062, + "l0_per_token_0.001": 125.63610112304687, + "l0_per_token_0.01": 125.63594052734375, + "l0_per_token_0.05": 125.635241796875, + "l0_per_token_0.1": 125.63404838867187, + "l0_per_token_0.5": 125.52499116210937, + "l0_per_token_1.0": 93.74265239257812, + "l1_per_token": 390.88694813323394, + "r2_score": 0.9410739181284478, + "l2_loss": 1.3819600468851936 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/config.json b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1cf101828ded1fb2652cef2e99dc904e97edaf17 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 2.95, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 5, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..a4d934011b702d3d097cd41c2f99f2f9535143df --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 125.77726040039063, + "l0_per_token_0.001": 125.77724150390625, + "l0_per_token_0.01": 125.77699580078125, + "l0_per_token_0.05": 125.77617680664062, + "l0_per_token_0.1": 125.774969140625, + "l0_per_token_0.5": 125.6611578125, + "l0_per_token_1.0": 93.548204296875, + "l1_per_token": 390.5783639531069, + "r2_score": 0.9409605591020928, + "l2_loss": 1.384618591970846 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_32/code_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x4/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..7e21f47ae6eb0dbdc96a3e4552ec3384c396a6a5 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 33.676136666666665, + "recon_r2": 0.8975673795678348, + "l1_per_token": 253.82649805907457 + }, + "seed1": { + "l0": 33.846435, + "recon_r2": 0.8977308049876321, + "l1_per_token": 254.91719231354833 + }, + "seed2": { + "l0": 33.651356666666665, + "recon_r2": 0.8976816471087172, + "l1_per_token": 255.21824300224597 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.8431015566494878, + "mcc_abs": 0.8431068267004274, + "act_iou": 0.5771575229163576, + "act_iou_n_pairs": 3072, + "input_r2": 0.9810264992004553, + "matched_mcc_signed": { + "0.6": 0.8933773801726083, + "0.7": 0.9059894971164321, + "0.8": 0.9244591009457132, + "0.9": 0.9450882109904006 + }, + "matched_mcc_abs": { + "0.6": 0.8941537238424455, + "0.7": 0.9051395484599205, + "0.8": 0.9227457160326656, + "0.9": 0.9450882109904014 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.8272049425533208, + "mcc_abs": 0.8272087362405648, + "act_iou": 0.5592969803272019, + "act_iou_n_pairs": 3072, + "input_r2": 0.9800477389075511, + "matched_mcc_signed": { + "0.6": 0.8880478491749985, + "0.7": 0.9024123006077128, + "0.8": 0.9184008541940856, + "0.9": 0.9426502364837515 + }, + "matched_mcc_abs": { + "0.6": 0.8882316570609861, + "0.7": 0.9017869623984147, + "0.8": 0.9181568606864884, + "0.9": 0.9426502364837515 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.8336904002024171, + "mcc_abs": 0.8337068280859405, + "act_iou": 0.5680103794159135, + "act_iou_n_pairs": 3072, + "input_r2": 0.9805106627129005, + "matched_mcc_signed": { + "0.6": 0.8906078080022163, + "0.7": 0.9041514540728112, + "0.8": 0.9187560352020746, + "0.9": 0.9430678425864627 + }, + "matched_mcc_abs": { + "0.6": 0.8897145007397163, + "0.7": 0.9036372232099904, + "0.8": 0.917934181590078, + "0.9": 0.9430055361585956 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 33.724642777777774, + "l0_std": 0.10620038203966387, + "recon_r2_mean": 0.8976599438880614, + "recon_r2_std": 8.384652228391995e-05, + "l1_per_token_mean": 254.65397779162296, + "l1_per_token_std": 0.7322567241924753, + "input_r2_mean": 0.9805283002736357, + "input_r2_std": 0.0004896184641156311, + "mcc_signed_mean": 0.8346656331350752, + "mcc_signed_std": 0.007993052882680055, + "mcc_abs_mean": 0.8346741303423109, + "mcc_abs_std": 0.007993064200222446, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8906776791166077, + "0.7": 0.904184417265652, + "0.8": 0.9205386634472911, + "0.9": 0.9436020966868716 + }, + "matched_mcc_signed_band": 0.9147507141291056, + "matched_mcc_abs_per_threshold": { + "0.6": 0.890699960547716, + "0.7": 0.903521244689442, + "0.8": 0.919612252769744, + "0.9": 0.9435813278775829 + }, + "matched_mcc_abs_band": 0.9143536964711212, + "act_iou_mean": 0.568154960886491, + "act_iou_std": 0.008931149043991313, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_32/dict_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x4/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..fba76626587f8943c3eb19b5ea547441a838217f --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_32/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 34, + "l0_raw": 34.167963525390626, + "recon_r2": 0.8977384601466455, + "l1_per_token": 255.04705028489641, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 20, + "dense_rate": 0.006510416666666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 313, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 149, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 164, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 4, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.37319397926330566, + "n_dup_abs": { + "0.5": 328, + "0.6": 40, + "0.7": 26, + "0.8": 8, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 155, + "0.6": 20, + "0.7": 13, + "0.8": 4, + "0.9": 0 + }, + "n_dup_removed_abs": { + "0.5": 173, + "0.6": 20, + "0.7": 13, + "0.8": 4, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.37956738471984863, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 34, + "l0_raw": 34.20432915039063, + "recon_r2": 0.8978192650851368, + "l1_per_token": 255.75038515835834, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 21, + "dense_rate": 0.0068359375, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 302, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 146, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 156, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 4, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.37258124351501465, + "n_dup_abs": { + "0.5": 321, + "0.6": 36, + "0.7": 20, + "0.8": 4, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 155, + "0.6": 18, + "0.7": 10, + "0.8": 2, + "0.9": 0 + }, + "n_dup_removed_abs": { + "0.5": 166, + "0.6": 18, + "0.7": 10, + "0.8": 2, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.3790799081325531, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 34, + "l0_raw": 34.06631484375, + "recon_r2": 0.8978286378898942, + "l1_per_token": 256.2946467344169, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 20, + "dense_rate": 0.006510416666666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 288, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 136, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 152, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 4, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.37247905135154724, + "n_dup_abs": { + "0.5": 305, + "0.6": 32, + "0.7": 18, + "0.8": 8, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 143, + "0.6": 16, + "0.7": 9, + "0.8": 4, + "0.9": 0 + }, + "n_dup_removed_abs": { + "0.5": 162, + "0.6": 16, + "0.7": 9, + "0.8": 4, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.37893176078796387, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8927777409553528, + "dcs_abs": 0.8932483196258545, + "frac_sign_flipped": 0.009440104166666666, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8908541202545166, + "0.6": 0.8927777409553528, + "0.7": 0.8927777409553528, + "0.8": 0.8927777409553528, + "0.9": 0.8927777409553528 + }, + "unique_frac_signed": { + "0.5": 0.9479166666666666, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.8444554681579272, + "0.6": 0.8927777409553528, + "0.7": 0.8927777409553528, + "0.8": 0.8927777409553528, + "0.9": 0.8927777409553528 + }, + "matched_frac_signed": { + "0.5": 0.8863932291666666, + "0.6": 0.9300130208333334, + "0.7": 0.8990885416666666, + "0.8": 0.8447265625, + "0.9": 0.75 + }, + "matched_cos_abs": { + "0.5": 0.8912460207939148, + "0.6": 0.8940423130989075, + "0.7": 0.8936648368835449, + "0.8": 0.8935339450836182, + "0.9": 0.8932483792304993 + }, + "unique_frac_abs": { + "0.5": 0.94482421875, + "0.6": 0.9938151041666666, + "0.7": 0.9962565104166666, + "0.8": 0.9990234375, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.8420708253106568, + "0.6": 0.8885127545217983, + "0.7": 0.8903194118756801, + "0.8": 0.8926613533403724, + "0.9": 0.8932483792304993 + }, + "matched_frac_abs": { + "0.5": 0.8834635416666666, + "0.6": 0.9231770833333334, + "0.7": 0.8955078125, + "0.8": 0.8470052083333334, + "0.9": 0.75 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.882415771484375, + "dcs_abs": 0.8828290104866028, + "frac_sign_flipped": 0.0078125, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.881562352180481, + "0.6": 0.882415771484375, + "0.7": 0.882415771484375, + "0.8": 0.882415771484375, + "0.9": 0.882415771484375 + }, + "unique_frac_signed": { + "0.5": 0.9485677083333334, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.8362215801607817, + "0.6": 0.882415771484375, + "0.7": 0.882415771484375, + "0.8": 0.882415771484375, + "0.9": 0.882415771484375 + }, + "matched_frac_signed": { + "0.5": 0.8798828125, + "0.6": 0.9108072916666666, + "0.7": 0.8782552083333334, + "0.8": 0.8317057291666666, + "0.9": 0.7233072916666666 + }, + "matched_cos_abs": { + "0.5": 0.8822004199028015, + "0.6": 0.8837887644767761, + "0.7": 0.8835625648498535, + "0.8": 0.8828677535057068, + "0.9": 0.8828290104866028 + }, + "unique_frac_abs": { + "0.5": 0.9454752604166666, + "0.6": 0.994140625, + "0.7": 0.9964192708333334, + "0.8": 0.9986979166666666, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.8340986717472939, + "0.6": 0.87861031468492, + "0.7": 0.8803987666033208, + "0.8": 0.8817181861183295, + "0.9": 0.8828290104866028 + }, + "matched_frac_abs": { + "0.5": 0.876953125, + "0.6": 0.9049479166666666, + "0.7": 0.8753255208333334, + "0.8": 0.8304036458333334, + "0.9": 0.7233072916666666 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8868472576141357, + "dcs_abs": 0.8873586058616638, + "frac_sign_flipped": 0.008463541666666666, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.884628176689148, + "0.6": 0.8868473172187805, + "0.7": 0.8868473172187805, + "0.8": 0.8868473172187805, + "0.9": 0.8868473172187805 + }, + "unique_frac_signed": { + "0.5": 0.9498697916666666, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.8402815818941841, + "0.6": 0.8868473172187805, + "0.7": 0.8868473172187805, + "0.8": 0.8868473172187805, + "0.9": 0.8868473172187805 + }, + "matched_frac_signed": { + "0.5": 0.8815104166666666, + "0.6": 0.9163411458333334, + "0.7": 0.8857421875, + "0.8": 0.841796875, + "0.9": 0.7317708333333334 + }, + "matched_cos_abs": { + "0.5": 0.8856135010719299, + "0.6": 0.8882299661636353, + "0.7": 0.8876924514770508, + "0.8": 0.8877029418945312, + "0.9": 0.8873586058616638 + }, + "unique_frac_abs": { + "0.5": 0.9466145833333334, + "0.6": 0.9944661458333334, + "0.7": 0.9969075520833334, + "0.8": 0.9990234375, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.8383346553115796, + "0.6": 0.8833146310644224, + "0.7": 0.8849473088048398, + "0.8": 0.8868360444903374, + "0.9": 0.8873586058616638 + }, + "matched_frac_abs": { + "0.5": 0.8785807291666666, + "0.6": 0.9108072916666666, + "0.7": 0.8834635416666666, + "0.8": 0.841796875, + "0.9": 0.7320963541666666 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 20, + "n_dense_std": 0.5773502691896258, + "l0_mean": 34, + "l0_std": 0.07153421325772635, + "recon_r2_mean": 0.8977954543738922, + "recon_r2_std": 4.9580427810158145e-05, + "l1_per_token_mean": 255.69736072589058, + "l1_per_token_std": 0.6254861453644979, + "babel_mu1_signed_mean": 0.37275142470995587, + "babel_mu1_signed_std": 0.00038665450868511356, + "n_dup_signed_per_threshold": { + "0.5": 301.0, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.0, + "n_dup_signed_band_std": 0.0, + "n_dup_clusters_signed_per_threshold": { + "0.5": 143.66666666666666, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.0, + "n_dup_clusters_signed_band_std": 0.0, + "largest_dup_signed_per_threshold": { + "0.5": 4.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.0, + "largest_dup_signed_band_std": 0.0, + "dcs_signed_mean": 0.8873469233512878, + "dcs_signed_std": 0.005199024159977689, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.8403195434042976, + "0.6": 0.8873469432195028, + "0.7": 0.8873469432195028, + "0.8": 0.8873469432195028, + "0.9": 0.8873469432195028 + }, + "dedup_dcs_signed_band": 0.8873469432195028, + "dedup_dcs_signed_band_std": 0.005199021295861095, + "matched_cos_signed_per_threshold": { + "0.5": 0.8856815497080485, + "0.6": 0.8873469432195028, + "0.7": 0.8873469432195028, + "0.8": 0.8873469432195028, + "0.9": 0.8873469432195028 + }, + "matched_cos_signed_band": 0.8873469432195028, + "matched_cos_signed_band_std": 0.005199021295861095, + "unique_frac_signed_per_threshold": { + "0.5": 0.9487847222222222, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 1.0, + "unique_frac_signed_band_std": 0.0, + "matched_frac_signed_per_threshold": { + "0.5": 0.882595486111111, + "0.6": 0.9190538194444445, + "0.7": 0.8876953125, + "0.8": 0.8394097222222222, + "0.9": 0.7350260416666666 + }, + "matched_frac_signed_band": 0.8452962239583334, + "matched_frac_signed_band_std": 0.010040813974395683, + "babel_mu1_abs_mean": 0.3791930178801219, + "babel_mu1_abs_std": 0.00033256549271153314, + "n_dup_abs_per_threshold": { + "0.5": 318.0, + "0.6": 36.0, + "0.7": 21.333333333333332, + "0.8": 6.666666666666667, + "0.9": 0.0 + }, + "n_dup_abs_band": 15.999999999999998, + "n_dup_abs_band_std": 2.179449471770337, + "n_dup_clusters_abs_per_threshold": { + "0.5": 151.0, + "0.6": 18.0, + "0.7": 10.666666666666666, + "0.8": 3.3333333333333335, + "0.9": 0.0 + }, + "n_dup_clusters_abs_band": 7.999999999999999, + "n_dup_clusters_abs_band_std": 1.0897247358851685, + "largest_dup_abs_per_threshold": { + "0.5": 4.0, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 1.0 + }, + "largest_dup_abs_band": 1.75, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.8878119786580404, + "dcs_abs_std": 0.005224429241954978, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.8381680507898434, + "0.6": 0.8834792334237136, + "0.7": 0.8852218290946136, + "0.8": 0.8870718613163464, + "0.9": 0.8878119985262553 + }, + "dedup_dcs_abs_band": 0.8858962305902321, + "dedup_dcs_abs_band_std": 0.005153995391320634, + "matched_cos_abs_per_threshold": { + "0.5": 0.8863533139228821, + "0.6": 0.888687014579773, + "0.7": 0.8883066177368164, + "0.8": 0.8880348801612854, + "0.9": 0.8878119985262553 + }, + "matched_cos_abs_band": 0.8882101277510325, + "matched_cos_abs_band_std": 0.005195743964577689, + "unique_frac_abs_per_threshold": { + "0.5": 0.9456380208333334, + "0.6": 0.994140625, + "0.7": 0.9965277777777778, + "0.8": 0.9989149305555555, + "0.9": 1.0 + }, + "unique_frac_abs_band": 0.9973958333333333, + "unique_frac_abs_band_std": 0.00017736405206471122, + "matched_frac_abs_per_threshold": { + "0.5": 0.879665798611111, + "0.6": 0.9129774305555555, + "0.7": 0.884765625, + "0.8": 0.8397352430555557, + "0.9": 0.735134548611111 + }, + "matched_frac_abs_band": 0.8431532118055556, + "matched_frac_abs_band_std": 0.01025853397418673, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 3072 + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/config.json b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac604db86ef545c68e6e53d67ed78486d69d766 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 7.1, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 15, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e8fce6f947238f3c4f3fadaf73ff546c7c3c80 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 34.167963525390626, + "l0_per_token_0.001": 34.167961572265625, + "l0_per_token_0.01": 34.167939111328124, + "l0_per_token_0.05": 34.167740185546876, + "l0_per_token_0.1": 34.16754931640625, + "l0_per_token_0.5": 34.164538037109374, + "l0_per_token_1.0": 34.15486186523437, + "l1_per_token": 255.04705028489641, + "r2_score": 0.8977384601466455, + "l2_loss": 2.3982820157353704 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_32/seed1/config.json b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..65176792143e871106ca8f733b6d2a829660004a --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed1/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 7.1, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 15, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e688beefac3bd66e124b9ab9c71300d57b191ba4 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 34.20432915039063, + "l0_per_token_0.001": 34.204326708984375, + "l0_per_token_0.01": 34.20431630859375, + "l0_per_token_0.05": 34.20423115234375, + "l0_per_token_0.1": 34.204119677734376, + "l0_per_token_0.5": 34.20169057617188, + "l0_per_token_1.0": 34.19195380859375, + "l1_per_token": 255.75038515835834, + "r2_score": 0.8978192650851368, + "l2_loss": 2.396386943247276 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_32/seed2/config.json b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d9669b860060b4df3b6d77caa83a58c4a8083565 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed2/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 7.1, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 15, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..c4b81e5c368f36cb21cae2a84ff0605633693a17 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 34.06631484375, + "l0_per_token_0.001": 34.06631064453125, + "l0_per_token_0.01": 34.06628505859375, + "l0_per_token_0.05": 34.0661236328125, + "l0_per_token_0.1": 34.06596328125, + "l0_per_token_0.5": 34.06256220703125, + "l0_per_token_1.0": 34.0512826171875, + "l1_per_token": 256.2946467344169, + "r2_score": 0.8978286378898942, + "l2_loss": 2.3961671097603374 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_64/code_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x4/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..2e5a2555e4d761e0c4e342413e387ac9fe64b9ed --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 62.36521833333333, + "recon_r2": 0.9044162366566666, + "l1_per_token": 247.6988576809062 + }, + "seed1": { + "l0": 62.51880666666667, + "recon_r2": 0.904503817848508, + "l1_per_token": 248.12824340134074 + }, + "seed2": { + "l0": 63.982841666666666, + "recon_r2": 0.9054041878237568, + "l1_per_token": 251.28907775817112 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.8634766960621616, + "mcc_abs": 0.8634845175466953, + "act_iou": 0.5746686419365702, + "act_iou_n_pairs": 3072, + "input_r2": 0.989327945689063, + "matched_mcc_signed": { + "0.6": 0.9134913967192205, + "0.7": 0.9252022219417604, + "0.8": 0.9370912647721716, + "0.9": 0.9566270459422009 + }, + "matched_mcc_abs": { + "0.6": 0.9150917228968076, + "0.7": 0.9244284601768906, + "0.8": 0.9358242983753443, + "0.9": 0.9567076409834316 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.8583650938311802, + "mcc_abs": 0.8583678619967866, + "act_iou": 0.5630731886620662, + "act_iou_n_pairs": 3072, + "input_r2": 0.9890714353133181, + "matched_mcc_signed": { + "0.6": 0.9109704713568352, + "0.7": 0.9203388907479161, + "0.8": 0.935132060745531, + "0.9": 0.9529659558466477 + }, + "matched_mcc_abs": { + "0.6": 0.9119654775909898, + "0.7": 0.9187571705839118, + "0.8": 0.9346026061389349, + "0.9": 0.9529724205683011 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.8584759416961827, + "mcc_abs": 0.8584825912519548, + "act_iou": 0.5619040753940389, + "act_iou_n_pairs": 3072, + "input_r2": 0.9889861899942557, + "matched_mcc_signed": { + "0.6": 0.910237064013519, + "0.7": 0.921101376898798, + "0.8": 0.9361787460504697, + "0.9": 0.9550780387628612 + }, + "matched_mcc_abs": { + "0.6": 0.9102880283579601, + "0.7": 0.9206434863791034, + "0.8": 0.9353038418193748, + "0.9": 0.9550780387628608 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 62.955622222222225, + "l0_std": 0.8929065932770298, + "recon_r2_mean": 0.9047747474429771, + "recon_r2_std": 0.000546867452869624, + "l1_per_token_mean": 249.03872628013937, + "l1_per_token_std": 1.9606515129512647, + "input_r2_mean": 0.9891285236655456, + "input_r2_std": 0.0001778863364717702, + "mcc_signed_mean": 0.8601059105298415, + "mcc_signed_std": 0.002919711996236578, + "mcc_abs_mean": 0.8601116569318122, + "mcc_abs_std": 0.0029215462081010964, + "matched_mcc_signed_per_threshold": { + "0.6": 0.9115663106965249, + "0.7": 0.9222141631961582, + "0.8": 0.9361340238560575, + "0.9": 0.95489034685057 + }, + "matched_mcc_signed_band": 0.9312012111498277, + "matched_mcc_abs_per_threshold": { + "0.6": 0.9124484096152524, + "0.7": 0.9212763723799685, + "0.8": 0.935243582111218, + "0.9": 0.9549193667715312 + }, + "matched_mcc_abs_band": 0.9309719327194925, + "act_iou_mean": 0.5665486353308918, + "act_iou_std": 0.0070563862504518125, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_64/dict_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x4/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..6300b589959e08a07fedd772e78f4943c32b07c8 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_64/dict_metrics_step_final.json @@ -0,0 +1,550 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 61, + "l0_raw": 61.03881782226563, + "recon_r2": 0.9040433110166821, + "l1_per_token": 246.4012132094577, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 9, + "dense_rate": 0.0029296875, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 656, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 293, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 5, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.4130684435367584, + "n_dup_abs": { + "0.5": 670, + "0.6": 86, + "0.7": 36, + "0.8": 6, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 302, + "0.6": 43, + "0.7": 18, + "0.8": 3, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 5, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.42120203375816345 + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 61, + "l0_raw": 61.13214165039062, + "recon_r2": 0.9041362180298654, + "l1_per_token": 246.7655775300305, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 5, + "dense_rate": 0.0016276041666666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 651, + "0.6": 3, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 291, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 3, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.4129961431026459, + "n_dup_abs": { + "0.5": 666, + "0.6": 91, + "0.7": 42, + "0.8": 14, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 305, + "0.6": 45, + "0.7": 21, + "0.8": 7, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 3, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.4220966398715973 + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 64, + "l0_raw": 63.96635864257812, + "recon_r2": 0.9052535313710259, + "l1_per_token": 251.18522851969087, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 15, + "dense_rate": 0.0048828125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 570, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 253, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 5, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.4041856825351715, + "n_dup_abs": { + "0.5": 574, + "0.6": 88, + "0.7": 36, + "0.8": 8, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 258, + "0.6": 44, + "0.7": 18, + "0.8": 4, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.4124111235141754 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.9081348776817322, + "dcs_abs": 0.9085099101066589, + "frac_sign_flipped": 0.008138020833333334, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.9051156044006348, + "0.6": 0.9084225296974182, + "0.7": 0.9081348776817322, + "0.8": 0.9081348776817322, + "0.9": 0.9081348776817322 + }, + "unique_frac_signed": { + "0.5": 0.88232421875, + "0.6": 0.99951171875, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7986054185312241, + "0.6": 0.9079789640090894, + "0.7": 0.9081348776817322, + "0.8": 0.9081348776817322, + "0.9": 0.9081348776817322 + }, + "matched_frac_signed": { + "0.5": 0.8255208333333334, + "0.6": 0.9329427083333334, + "0.7": 0.9104817708333334, + "0.8": 0.8805338541666666, + "0.9": 0.7972005208333334 + }, + "matched_cos_abs": { + "0.5": 0.9061989784240723, + "0.6": 0.9106427431106567, + "0.7": 0.9095739722251892, + "0.8": 0.9092931747436523, + "0.9": 0.9085099697113037 + }, + "unique_frac_abs": { + "0.5": 0.88134765625, + "0.6": 0.9855143229166666, + "0.7": 0.99365234375, + "0.8": 0.9983723958333334, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.7986763457302004, + "0.6": 0.8974514663956749, + "0.7": 0.9038003093155567, + "0.8": 0.907813205383718, + "0.9": 0.9085099697113037 + }, + "matched_frac_abs": { + "0.5": 0.8229166666666666, + "0.6": 0.919921875, + "0.7": 0.9059244791666666, + "0.8": 0.8802083333333334, + "0.9": 0.7965494791666666 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.9031007885932922, + "dcs_abs": 0.9035313129425049, + "frac_sign_flipped": 0.010091145833333334, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.9064162969589233, + "0.6": 0.9030746817588806, + "0.7": 0.9031007885932922, + "0.8": 0.9031007885932922, + "0.9": 0.9031007885932922 + }, + "unique_frac_signed": { + "0.5": 0.8893229166666666, + "0.6": 0.9998372395833334, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.8060967849257091, + "0.6": 0.9029276969473965, + "0.7": 0.9031007885932922, + "0.8": 0.9031007885932922, + "0.9": 0.9031007885932922 + }, + "matched_frac_signed": { + "0.5": 0.8287760416666666, + "0.6": 0.9267578125, + "0.7": 0.90625, + "0.8": 0.8658854166666666, + "0.9": 0.7838541666666666 + }, + "matched_cos_abs": { + "0.5": 0.9077838063240051, + "0.6": 0.9051803946495056, + "0.7": 0.9040634632110596, + "0.8": 0.9038311243057251, + "0.9": 0.9035313129425049 + }, + "unique_frac_abs": { + "0.5": 0.888671875, + "0.6": 0.98583984375, + "0.7": 0.994140625, + "0.8": 0.9988606770833334, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.8067219372605905, + "0.6": 0.892362898826832, + "0.7": 0.8987662163563073, + "0.8": 0.902801368793007, + "0.9": 0.9035313129425049 + }, + "matched_frac_abs": { + "0.5": 0.828125, + "0.6": 0.9127604166666666, + "0.7": 0.9013671875, + "0.8": 0.865234375, + "0.9": 0.7835286458333334 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.9020555019378662, + "dcs_abs": 0.9023888111114502, + "frac_sign_flipped": 0.005859375, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.9052185416221619, + "0.6": 0.902321457862854, + "0.7": 0.9020554423332214, + "0.8": 0.9020554423332214, + "0.9": 0.9020554423332214 + }, + "unique_frac_signed": { + "0.5": 0.8898111979166666, + "0.6": 0.9996744791666666, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.8054735948971938, + "0.6": 0.9020277334299559, + "0.7": 0.9020554423332214, + "0.8": 0.9020554423332214, + "0.9": 0.9020554423332214 + }, + "matched_frac_signed": { + "0.5": 0.8310546875, + "0.6": 0.9283854166666666, + "0.7": 0.9049479166666666, + "0.8": 0.8619791666666666, + "0.9": 0.7799479166666666 + }, + "matched_cos_abs": { + "0.5": 0.9060282707214355, + "0.6": 0.9041953682899475, + "0.7": 0.9032856822013855, + "0.8": 0.9029776453971863, + "0.9": 0.9023887515068054 + }, + "unique_frac_abs": { + "0.5": 0.8898111979166666, + "0.6": 0.9853515625, + "0.7": 0.99365234375, + "0.8": 0.9982096354166666, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.8061941009170065, + "0.6": 0.8909503189497627, + "0.7": 0.8975519351952244, + "0.8": 0.9013609862013254, + "0.9": 0.9023887515068054 + }, + "matched_frac_abs": { + "0.5": 0.8304036458333334, + "0.6": 0.9147135416666666, + "0.7": 0.8990885416666666, + "0.8": 0.8616536458333334, + "0.9": 0.7799479166666666 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 10, + "n_dense_std": 5.033222956847166, + "l0_mean": 62, + "l0_std": 1.6639306150546167, + "recon_r2_mean": 0.9044776868058578, + "recon_r2_std": 0.000673505026150752, + "l1_per_token_mean": 248.1173397530597, + "l1_per_token_std": 2.663108419280552, + "babel_mu1_signed_mean": 0.41008342305819195, + "babel_mu1_signed_std": 0.005107721047184464, + "n_dup_signed_per_threshold": { + "0.5": 625.6666666666666, + "0.6": 1.6666666666666667, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.4166666666666667, + "n_dup_signed_band_std": 0.3818813079129867, + "n_dup_clusters_signed_per_threshold": { + "0.5": 279.0, + "0.6": 0.6666666666666666, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.16666666666666666, + "n_dup_clusters_signed_band_std": 0.14433756729740646, + "largest_dup_signed_per_threshold": { + "0.5": 5.333333333333333, + "0.6": 2.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.25, + "largest_dup_signed_band_std": 0.25, + "dcs_signed_mean": 0.9044303894042969, + "dcs_signed_std": 0.0032504739802442064, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.803391932784709, + "0.6": 0.9043114647954806, + "0.7": 0.9044303695360819, + "0.8": 0.9044303695360819, + "0.9": 0.9044303695360819 + }, + "dedup_dcs_signed_band": 0.9044006433509315, + "dedup_dcs_signed_band_std": 0.003239707970591871, + "matched_cos_signed_per_threshold": { + "0.5": 0.9055834809939066, + "0.6": 0.9046062231063843, + "0.7": 0.9044303695360819, + "0.8": 0.9044303695360819, + "0.9": 0.9044303695360819 + }, + "matched_cos_signed_band": 0.9044743329286574, + "matched_cos_signed_band_std": 0.0032687581924785067, + "unique_frac_signed_per_threshold": { + "0.5": 0.8871527777777777, + "0.6": 0.9996744791666666, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9999186197916666, + "unique_frac_signed_band_std": 4.069010416668517e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.8284505208333334, + "0.6": 0.9293619791666666, + "0.7": 0.9072265625, + "0.8": 0.8694661458333334, + "0.9": 0.7870008680555555 + }, + "matched_frac_signed_band": 0.8732638888888888, + "matched_frac_signed_band_std": 0.006156095855326547, + "babel_mu1_abs_mean": 0.41856993238131207, + "babel_mu1_abs_std": 0.0053524083384014105, + "n_dup_abs_per_threshold": { + "0.5": 636.6666666666666, + "0.6": 88.33333333333333, + "0.7": 38.0, + "0.8": 9.333333333333334, + "0.9": 0.0 + }, + "n_dup_abs_band": 33.916666666666664, + "n_dup_abs_band_std": 2.504163200219453, + "n_dup_clusters_abs_per_threshold": { + "0.5": 288.3333333333333, + "0.6": 44.0, + "0.7": 19.0, + "0.8": 4.666666666666667, + "0.9": 0.0 + }, + "n_dup_clusters_abs_band": 16.916666666666668, + "n_dup_clusters_abs_band_std": 1.181453906563152, + "largest_dup_abs_per_threshold": { + "0.5": 5.0, + "0.6": 2.3333333333333335, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 1.0 + }, + "largest_dup_abs_band": 1.8333333333333335, + "largest_dup_abs_band_std": 0.14433756729740646, + "dcs_abs_mean": 0.9048100113868713, + "dcs_abs_std": 0.0032547297139709245, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.8038641279692658, + "0.6": 0.8935882280574231, + "0.7": 0.9000394869556961, + "0.8": 0.9039918534593502, + "0.9": 0.9048100113868713 + }, + "dedup_dcs_abs_band": 0.9006073949648352, + "dedup_dcs_abs_band_std": 0.003343110574794992, + "matched_cos_abs_per_threshold": { + "0.5": 0.906670351823171, + "0.6": 0.9066728353500366, + "0.7": 0.9056410392125448, + "0.8": 0.9053673148155212, + "0.9": 0.9048100113868713 + }, + "matched_cos_abs_band": 0.9056228001912435, + "matched_cos_abs_band_std": 0.0033947263548461146, + "unique_frac_abs_per_threshold": { + "0.5": 0.8866102430555555, + "0.6": 0.9855685763888888, + "0.7": 0.9938151041666666, + "0.8": 0.9984809027777778, + "0.9": 1.0 + }, + "unique_frac_abs_band": 0.9944661458333333, + "unique_frac_abs_band_std": 0.00021531179289266115, + "matched_frac_abs_per_threshold": { + "0.5": 0.8271484375, + "0.6": 0.915798611111111, + "0.7": 0.902126736111111, + "0.8": 0.8690321180555557, + "0.9": 0.7866753472222222 + }, + "matched_frac_abs_band": 0.868408203125, + "matched_frac_abs_band_std": 0.00634191530586557, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 3072 + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_64/seed0/config.json b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..76a689e1b3a6de3411afabdac7f4bfc262be8396 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed0/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 4.6, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 5, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b4758a00071f8264b474194837a348c5eff46d9c --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 61.03881782226563, + "l0_per_token_0.001": 61.03880678710937, + "l0_per_token_0.01": 61.038694189453125, + "l0_per_token_0.05": 61.03797119140625, + "l0_per_token_0.1": 61.03666586914063, + "l0_per_token_0.5": 60.516551025390626, + "l0_per_token_1.0": 46.52499819335937, + "l1_per_token": 246.4012132094577, + "r2_score": 0.9040433110166821, + "l2_loss": 2.2504179167281992 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_64/seed1/config.json b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fbb899280fbf679689c670afb4ea743d3f0d99d9 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed1/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 4.6, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 5, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..55dea1d4d21068d4c1540f951fc750c8da6bd84c --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 61.13214165039062, + "l0_per_token_0.001": 61.132130615234374, + "l0_per_token_0.01": 61.13199038085938, + "l0_per_token_0.05": 61.13125595703125, + "l0_per_token_0.1": 61.1297873046875, + "l0_per_token_0.5": 60.615798486328124, + "l0_per_token_1.0": 46.63437973632813, + "l1_per_token": 246.7655775300305, + "r2_score": 0.9041362180298654, + "l2_loss": 2.2482390211319387 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_64/seed2/config.json b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..34b345e09034bcb6126b47e9eae9aac5087eb326 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed2/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 4.6, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 5, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..409cedec344a6949a8883811681fe0512abe4ab9 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.96635864257812, + "l0_per_token_0.001": 63.966344921875, + "l0_per_token_0.01": 63.96624194335938, + "l0_per_token_0.05": 63.965510498046875, + "l0_per_token_0.1": 63.96405166015625, + "l0_per_token_0.5": 63.443435498046874, + "l0_per_token_1.0": 49.04058618164063, + "l1_per_token": 251.18522851969087, + "r2_score": 0.9052535313710259, + "l2_loss": 2.2220353037237195 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..8050d46eebc38f6dfad35f53ad2b37df49594c4b --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_128/code_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x8/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..9ded3d527991c37df9101d81fcf765cc35fee967 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 134.04299, + "recon_r2": 0.940038611522527, + "l1_per_token": 311.7450949745843 + }, + "seed1": { + "l0": 134.51837833333335, + "recon_r2": 0.9400403907178778, + "l1_per_token": 312.49335202245345 + }, + "seed2": { + "l0": 134.34071, + "recon_r2": 0.9400340025984419, + "l1_per_token": 310.7476609769396 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.8384229365267496, + "mcc_abs": 0.8384239590946798, + "act_iou": 0.5034942920707556, + "act_iou_n_pairs": 6144, + "input_r2": 0.9915858769282769, + "matched_mcc_signed": { + "0.6": 0.8858232481128905, + "0.7": 0.9028940031926763, + "0.8": 0.9230262577339394, + "0.9": 0.9470610025210956 + }, + "matched_mcc_abs": { + "0.6": 0.8849166250991063, + "0.7": 0.9017930713889625, + "0.8": 0.9229868605527688, + "0.9": 0.947061002521095 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.8374360980313617, + "mcc_abs": 0.837438340604017, + "act_iou": 0.5029526480606462, + "act_iou_n_pairs": 6144, + "input_r2": 0.9915844677289133, + "matched_mcc_signed": { + "0.6": 0.883430855241725, + "0.7": 0.903302031787148, + "0.8": 0.9245031294761411, + "0.9": 0.9483537546560508 + }, + "matched_mcc_abs": { + "0.6": 0.8840461869524273, + "0.7": 0.9026825067923898, + "0.8": 0.9246546527705316, + "0.9": 0.9483449052823196 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.8360377771397633, + "mcc_abs": 0.8360386363907054, + "act_iou": 0.5025216473265913, + "act_iou_n_pairs": 6144, + "input_r2": 0.9916173075984112, + "matched_mcc_signed": { + "0.6": 0.884391533791699, + "0.7": 0.8998240838896734, + "0.8": 0.9222200341976664, + "0.9": 0.9464554100237893 + }, + "matched_mcc_abs": { + "0.6": 0.8846269087860724, + "0.7": 0.8994261429291149, + "0.8": 0.9221627640364992, + "0.9": 0.9464554100237893 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 134.30069277777778, + "l0_std": 0.24020730718100552, + "recon_r2_mean": 0.9400376682796155, + "recon_r2_std": 3.2968618221253185e-06, + "l1_per_token_mean": 311.6620359913258, + "l1_per_token_std": 0.8758044317207899, + "input_r2_mean": 0.9915958840852005, + "input_r2_std": 1.8566681163534308e-05, + "mcc_signed_mean": 0.8372989372326248, + "mcc_signed_std": 0.001198480762826311, + "mcc_abs_mean": 0.8373003120298007, + "mcc_abs_std": 0.00119863673229176, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8845485457154382, + "0.7": 0.9020067062898326, + "0.8": 0.9232498071359156, + "0.9": 0.9472900557336453 + }, + "matched_mcc_signed_band": 0.914273778718708, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8845299069458687, + "0.7": 0.901300573703489, + "0.8": 0.9232680924532666, + "0.9": 0.9472871059424013 + }, + "matched_mcc_abs_band": 0.9140964197612564, + "act_iou_mean": 0.5029895291526644, + "act_iou_std": 0.000487370096332269, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_128/dict_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x8/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..1d4f8ed74a1caffc1801ba98ba91f9e171f01fe7 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_128/dict_metrics_step_final.json @@ -0,0 +1,550 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 134, + "l0_raw": 134.105478125, + "recon_r2": 0.9399232212831633, + "l1_per_token": 311.7330372541613, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 25, + "dense_rate": 0.004069010416666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1262, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 577, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 5, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.41438618302345276, + "n_dup_abs": { + "0.5": 1349, + "0.6": 116, + "0.7": 32, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 621, + "0.6": 58, + "0.7": 16, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 5, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_abs": 0.42198315262794495 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 135, + "l0_raw": 134.57239296875, + "recon_r2": 0.9399400551368547, + "l1_per_token": 312.48225852616065, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 25, + "dense_rate": 0.004069010416666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1234, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 562, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 5, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.41367316246032715, + "n_dup_abs": { + "0.5": 1323, + "0.6": 126, + "0.7": 24, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 608, + "0.6": 63, + "0.7": 12, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 5, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_abs": 0.4214427173137665 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 134, + "l0_raw": 134.42412495117188, + "recon_r2": 0.9399258806606918, + "l1_per_token": 310.7607959557631, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 23, + "dense_rate": 0.0037434895833333335, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1278, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 593, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.41396212577819824, + "n_dup_abs": { + "0.5": 1356, + "0.6": 128, + "0.7": 40, + "0.8": 4, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 630, + "0.6": 64, + "0.7": 20, + "0.8": 2, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.42175570130348206 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8863566517829895, + "dcs_abs": 0.886591374874115, + "frac_sign_flipped": 0.00634765625, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8851808309555054, + "0.6": 0.8863566517829895, + "0.7": 0.8863566517829895, + "0.8": 0.8863566517829895, + "0.9": 0.8863566517829895 + }, + "unique_frac_signed": { + "0.5": 0.8895670572916666, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7874277069640812, + "0.6": 0.8863566517829895, + "0.7": 0.8863566517829895, + "0.8": 0.8863566517829895, + "0.9": 0.8863566517829895 + }, + "matched_frac_signed": { + "0.5": 0.8356119791666666, + "0.6": 0.9283854166666666, + "0.7": 0.8883463541666666, + "0.8": 0.8282877604166666, + "0.9": 0.7076822916666666 + }, + "matched_cos_abs": { + "0.5": 0.8865467309951782, + "0.6": 0.8888446688652039, + "0.7": 0.8871003985404968, + "0.8": 0.886591374874115, + "0.9": 0.886591374874115 + }, + "unique_frac_abs": { + "0.5": 0.882568359375, + "0.6": 0.9901529947916666, + "0.7": 0.9977213541666666, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.7824380938836839, + "0.6": 0.8800922107814888, + "0.7": 0.8850790109136142, + "0.8": 0.886591374874115, + "0.9": 0.886591374874115 + }, + "matched_frac_abs": { + "0.5": 0.8277994791666666, + "0.6": 0.9208984375, + "0.7": 0.8873697916666666, + "0.8": 0.8279622395833334, + "0.9": 0.7076822916666666 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8856496810913086, + "dcs_abs": 0.8859302401542664, + "frac_sign_flipped": 0.006673177083333333, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8828248381614685, + "0.6": 0.885744571685791, + "0.7": 0.8856496810913086, + "0.8": 0.8856496810913086, + "0.9": 0.8856496810913086 + }, + "unique_frac_signed": { + "0.5": 0.8885091145833334, + "0.6": 0.9999186197916666, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.784397915287021, + "0.6": 0.885672489608017, + "0.7": 0.8856496810913086, + "0.8": 0.8856496810913086, + "0.9": 0.8856496810913086 + }, + "matched_frac_signed": { + "0.5": 0.8336588541666666, + "0.6": 0.9288736979166666, + "0.7": 0.8844401041666666, + "0.8": 0.8216145833333334, + "0.9": 0.7068684895833334 + }, + "matched_cos_abs": { + "0.5": 0.884332001209259, + "0.6": 0.8878750205039978, + "0.7": 0.8865873217582703, + "0.8": 0.8860963582992554, + "0.9": 0.8859302401542664 + }, + "unique_frac_abs": { + "0.5": 0.8816731770833334, + "0.6": 0.9900716145833334, + "0.7": 0.9970703125, + "0.8": 0.9998372395833334, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.7796918051026296, + "0.6": 0.8790598550986033, + "0.7": 0.8839898979640566, + "0.8": 0.8859521368867718, + "0.9": 0.8859302401542664 + }, + "matched_frac_abs": { + "0.5": 0.8277994791666666, + "0.6": 0.9195963541666666, + "0.7": 0.8826497395833334, + "0.8": 0.8212890625, + "0.9": 0.70703125 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8852503895759583, + "dcs_abs": 0.8855137228965759, + "frac_sign_flipped": 0.005859375, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8831397891044617, + "0.6": 0.8853555917739868, + "0.7": 0.8852503895759583, + "0.8": 0.8852503895759583, + "0.9": 0.8852503895759583 + }, + "unique_frac_signed": { + "0.5": 0.8895670572916666, + "0.6": 0.9999186197916666, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.785612063370839, + "0.6": 0.8852835413514791, + "0.7": 0.8852503895759583, + "0.8": 0.8852503895759583, + "0.9": 0.8852503895759583 + }, + "matched_frac_signed": { + "0.5": 0.8370768229166666, + "0.6": 0.9264322916666666, + "0.7": 0.8902994791666666, + "0.8": 0.8235677083333334, + "0.9": 0.7034505208333334 + }, + "matched_cos_abs": { + "0.5": 0.8843286037445068, + "0.6": 0.8871275782585144, + "0.7": 0.8861605525016785, + "0.8": 0.8856523633003235, + "0.9": 0.8855137228965759 + }, + "unique_frac_abs": { + "0.5": 0.8827311197916666, + "0.6": 0.9896647135416666, + "0.7": 0.9973958333333334, + "0.8": 0.9998372395833334, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.7806243786471896, + "0.6": 0.8779588606121251, + "0.7": 0.8838528427295387, + "0.8": 0.8855082141526509, + "0.9": 0.8855137228965759 + }, + "matched_frac_abs": { + "0.5": 0.8299153645833334, + "0.6": 0.9173177083333334, + "0.7": 0.88916015625, + "0.8": 0.8240559895833334, + "0.9": 0.7034505208333334 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 24, + "n_dense_std": 1.1547005383792515, + "l0_mean": 134, + "l0_std": 0.23858215781991887, + "recon_r2_mean": 0.9399297190269033, + "recon_r2_std": 9.049555177257657e-06, + "l1_per_token_mean": 311.6586972453617, + "l1_per_token_std": 0.8631356631498691, + "babel_mu1_signed_mean": 0.41400715708732605, + "babel_mu1_signed_std": 0.00035863692637723985, + "n_dup_signed_per_threshold": { + "0.5": 1258.0, + "0.6": 0.6666666666666666, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.16666666666666666, + "n_dup_signed_band_std": 0.2886751345948129, + "n_dup_clusters_signed_per_threshold": { + "0.5": 577.3333333333334, + "0.6": 0.3333333333333333, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.08333333333333333, + "n_dup_clusters_signed_band_std": 0.14433756729740646, + "largest_dup_signed_per_threshold": { + "0.5": 5.333333333333333, + "0.6": 1.3333333333333333, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.0833333333333333, + "largest_dup_signed_band_std": 0.14433756729740646, + "dcs_signed_mean": 0.8857522408167521, + "dcs_signed_std": 0.0005602168246658618, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7858125618739805, + "0.6": 0.8857708942474952, + "0.7": 0.8857522408167521, + "0.8": 0.8857522408167521, + "0.9": 0.8857522408167521 + }, + "dedup_dcs_signed_band": 0.8857569041744379, + "dedup_dcs_signed_band_std": 0.0005559826874139636, + "matched_cos_signed_per_threshold": { + "0.5": 0.8837151527404785, + "0.6": 0.8858189384142557, + "0.7": 0.8857522408167521, + "0.8": 0.8857522408167521, + "0.9": 0.8857522408167521 + }, + "matched_cos_signed_band": 0.885768915216128, + "matched_cos_signed_band_std": 0.0005462793488635784, + "unique_frac_signed_per_threshold": { + "0.5": 0.8892144097222222, + "0.6": 0.9999457465277777, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9999864366319444, + "unique_frac_signed_band_std": 1.1746221297010822e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.83544921875, + "0.6": 0.9278971354166666, + "0.7": 0.8876953125, + "0.8": 0.8244900173611112, + "0.9": 0.7060004340277778 + }, + "matched_frac_signed_band": 0.8365207248263888, + "matched_frac_signed_band_std": 0.001453686835336672, + "babel_mu1_abs_mean": 0.4217271904150645, + "babel_mu1_abs_std": 0.00027134338995369597, + "n_dup_abs_per_threshold": { + "0.5": 1342.6666666666667, + "0.6": 123.33333333333333, + "0.7": 32.0, + "0.8": 1.3333333333333333, + "0.9": 0.0 + }, + "n_dup_abs_band": 39.166666666666664, + "n_dup_abs_band_std": 3.3291640592396963, + "n_dup_clusters_abs_per_threshold": { + "0.5": 619.6666666666666, + "0.6": 61.666666666666664, + "0.7": 16.0, + "0.8": 0.6666666666666666, + "0.9": 0.0 + }, + "n_dup_clusters_abs_band": 19.583333333333332, + "n_dup_clusters_abs_band_std": 1.6645820296198481, + "largest_dup_abs_per_threshold": { + "0.5": 5.333333333333333, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 1.3333333333333333, + "0.9": 1.0 + }, + "largest_dup_abs_band": 1.5833333333333333, + "largest_dup_abs_band_std": 0.14433756729740646, + "dcs_abs_mean": 0.8860117793083191, + "dcs_abs_std": 0.0005434334562816221, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.7809180925445011, + "0.6": 0.8790369754974057, + "0.7": 0.8843072505357364, + "0.8": 0.8860172419711793, + "0.9": 0.8860117793083191 + }, + "dedup_dcs_abs_band": 0.8838433118281601, + "dedup_dcs_abs_band_std": 0.0006966191582456482, + "matched_cos_abs_per_threshold": { + "0.5": 0.8850691119829813, + "0.6": 0.8879490892092387, + "0.7": 0.8866160909334818, + "0.8": 0.8861133654912313, + "0.9": 0.8860117793083191 + }, + "matched_cos_abs_band": 0.8866725812355677, + "matched_cos_abs_band_std": 0.0005858248140964629, + "unique_frac_abs_per_threshold": { + "0.5": 0.88232421875, + "0.6": 0.9899631076388888, + "0.7": 0.9973958333333334, + "0.8": 0.9998914930555557, + "0.9": 1.0 + }, + "unique_frac_abs_band": 0.9968126085069445, + "unique_frac_abs_band_std": 0.0001354640323583129, + "matched_frac_abs_per_threshold": { + "0.5": 0.8285047743055555, + "0.6": 0.9192708333333334, + "0.7": 0.8863932291666666, + "0.8": 0.824435763888889, + "0.9": 0.7060546875 + }, + "matched_frac_abs_band": 0.8340386284722222, + "matched_frac_abs_band_std": 0.0017331946566118155, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 6144 + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_128/seed0/config.json b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4e651652ad81f11007a0718f6c6b422972ae6458 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed0/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 3.4, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 5, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..ffdf2620e19109847ea4cb2379790850304cca4a --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 134.105478125, + "l0_per_token_0.001": 134.10542353515626, + "l0_per_token_0.01": 134.10485712890625, + "l0_per_token_0.05": 134.10195463867188, + "l0_per_token_0.1": 134.09474174804689, + "l0_per_token_0.5": 105.96406899414062, + "l0_per_token_1.0": 76.31803178710938, + "l1_per_token": 311.7330372541613, + "r2_score": 0.9399232212831633, + "l2_loss": 1.4089466887210842 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_128/seed1/config.json b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0d87498ee6a0d1f264548613ce02d01be57f748f --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed1/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 3.4, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 5, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..4144a603080cc145b583475a5f0136760757c1c7 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 134.57239296875, + "l0_per_token_0.001": 134.57229077148438, + "l0_per_token_0.01": 134.57172661132813, + "l0_per_token_0.05": 134.56812280273436, + "l0_per_token_0.1": 134.55980209960939, + "l0_per_token_0.5": 106.21981748046875, + "l0_per_token_1.0": 76.49339697265626, + "l1_per_token": 312.48225852616065, + "r2_score": 0.9399400551368547, + "l2_loss": 1.4085518938781578 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_128/seed2/config.json b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ea5bf627b5c1ad9b703175efee8607e61465fc86 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed2/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 3.4, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 5, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..5f927c1554a31ad31b6f2783b16810a546d5af7d --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 134.42412495117188, + "l0_per_token_0.001": 134.424066796875, + "l0_per_token_0.01": 134.42350556640625, + "l0_per_token_0.05": 134.42008095703125, + "l0_per_token_0.1": 134.41211494140626, + "l0_per_token_0.5": 106.0866099609375, + "l0_per_token_1.0": 76.39649951171874, + "l1_per_token": 310.7607959557631, + "r2_score": 0.9399258806606918, + "l2_loss": 1.4088843198450725 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_32/code_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x8/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..ee17e880565fae88cc630a7b8f0db31525c3f2da --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 36.670805, + "recon_r2": 0.899623732840085, + "l1_per_token": 201.2047141124633 + }, + "seed1": { + "l0": 36.30084333333333, + "recon_r2": 0.8997862483478682, + "l1_per_token": 201.40013700024232 + }, + "seed2": { + "l0": 36.762168333333335, + "recon_r2": 0.8996617654267405, + "l1_per_token": 201.0669341177959 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.8437479388097079, + "mcc_abs": 0.8437507393159468, + "act_iou": 0.5848594269768644, + "act_iou_n_pairs": 6144, + "input_r2": 0.9879807452796052, + "matched_mcc_signed": { + "0.6": 0.8898819057790719, + "0.7": 0.9045191503545211, + "0.8": 0.9253824682409457, + "0.9": 0.9521965717833563 + }, + "matched_mcc_abs": { + "0.6": 0.8905995166906227, + "0.7": 0.9046961772798713, + "0.8": 0.9252838156752777, + "0.9": 0.952196571783356 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.843593677253177, + "mcc_abs": 0.8435970200340567, + "act_iou": 0.5840609810021343, + "act_iou_n_pairs": 6144, + "input_r2": 0.9881299431194247, + "matched_mcc_signed": { + "0.6": 0.8887445069037262, + "0.7": 0.9044063761782095, + "0.8": 0.9256025467132135, + "0.9": 0.953030535005705 + }, + "matched_mcc_abs": { + "0.6": 0.889629912493724, + "0.7": 0.9037161536971099, + "0.8": 0.9253267699520208, + "0.9": 0.953042342695178 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.8430338653163654, + "mcc_abs": 0.8430357854860423, + "act_iou": 0.5838303397689568, + "act_iou_n_pairs": 6144, + "input_r2": 0.9880861110926863, + "matched_mcc_signed": { + "0.6": 0.8887483859200588, + "0.7": 0.9026673042911406, + "0.8": 0.9255174617458994, + "0.9": 0.9527756661779441 + }, + "matched_mcc_abs": { + "0.6": 0.8889278822383415, + "0.7": 0.9016362214852321, + "0.8": 0.9253134138206506, + "0.9": 0.952797553494978 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 36.57793888888889, + "l0_std": 0.24428113895863207, + "recon_r2_mean": 0.8996905822048978, + "recon_r2_std": 8.500369165537245e-05, + "l1_per_token_mean": 201.2239284101672, + "l1_per_token_std": 0.1674303799937144, + "input_r2_mean": 0.9880655998305721, + "input_r2_std": 7.668463181654194e-05, + "mcc_signed_mean": 0.8434584937930835, + "mcc_signed_std": 0.00037574082076047196, + "mcc_abs_mean": 0.8434611816120152, + "mcc_abs_std": 0.0003763360232568815, + "matched_mcc_signed_per_threshold": { + "0.6": 0.889124932867619, + "0.7": 0.9038642769412905, + "0.8": 0.9255008255666862, + "0.9": 0.9526675909890017 + }, + "matched_mcc_signed_band": 0.9177894065911493, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8897191038075626, + "0.7": 0.9033495174874044, + "0.8": 0.925307999815983, + "0.9": 0.9526788226578372 + }, + "matched_mcc_abs_band": 0.9177638609421968, + "act_iou_mean": 0.5842502492493186, + "act_iou_std": 0.0005400203444444569, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_32/dict_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x8/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..b18b2c614c8ed134a6841ae85308b295f997c3f7 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_32/dict_metrics_step_final.json @@ -0,0 +1,550 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 37, + "l0_raw": 36.65106376953125, + "recon_r2": 0.8994488106656073, + "l1_per_token": 201.06601714415362, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2031, + "0.6": 7, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 918, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 5, + "0.6": 5, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.453784316778183, + "n_dup_abs": { + "0.5": 2044, + "0.6": 97, + "0.7": 28, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 923, + "0.6": 47, + "0.7": 14, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 5, + "0.6": 5, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.45810261368751526 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 36, + "l0_raw": 36.2884943359375, + "recon_r2": 0.8996215318334677, + "l1_per_token": 201.2937255953762, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1991, + "0.6": 4, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 899, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.4538145065307617, + "n_dup_abs": { + "0.5": 2016, + "0.6": 94, + "0.7": 28, + "0.8": 4, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 910, + "0.6": 47, + "0.7": 14, + "0.8": 2, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.45827463269233704 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 37, + "l0_raw": 36.75276030273437, + "recon_r2": 0.8994905908240057, + "l1_per_token": 200.9609080229157, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2016, + "0.6": 7, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 914, + "0.6": 3, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 3, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.4538631737232208, + "n_dup_abs": { + "0.5": 2026, + "0.6": 91, + "0.7": 32, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 917, + "0.6": 45, + "0.7": 16, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 3, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_abs": 0.45807763934135437 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8967041969299316, + "dcs_abs": 0.896843433380127, + "frac_sign_flipped": 0.004557291666666667, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8961247801780701, + "0.6": 0.8965979814529419, + "0.7": 0.8967041969299316, + "0.8": 0.8967041969299316, + "0.9": 0.8967041969299316 + }, + "unique_frac_signed": { + "0.5": 0.820556640625, + "0.6": 0.9994303385416666, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7353211392037338, + "0.6": 0.8960872241392887, + "0.7": 0.8967041969299316, + "0.8": 0.8967041969299316, + "0.9": 0.8967041969299316 + }, + "matched_frac_signed": { + "0.5": 0.7747395833333334, + "0.6": 0.9342447916666666, + "0.7": 0.90576171875, + "0.8": 0.84912109375, + "0.9": 0.73046875 + }, + "matched_cos_abs": { + "0.5": 0.8965231776237488, + "0.6": 0.8977172374725342, + "0.7": 0.8969992399215698, + "0.8": 0.8969413042068481, + "0.9": 0.896843433380127 + }, + "unique_frac_abs": { + "0.5": 0.8187662760416666, + "0.6": 0.9921061197916666, + "0.7": 0.9977213541666666, + "0.8": 0.999755859375, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.7340429435280384, + "0.6": 0.89063076513897, + "0.7": 0.8949552963410193, + "0.8": 0.8967223243962508, + "0.9": 0.896843433380127 + }, + "matched_frac_abs": { + "0.5": 0.7737630208333334, + "0.6": 0.9275716145833334, + "0.7": 0.9036458333333334, + "0.8": 0.8494466145833334, + "0.9": 0.73046875 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.895737886428833, + "dcs_abs": 0.8958330750465393, + "frac_sign_flipped": 0.004069010416666667, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8952751755714417, + "0.6": 0.8954804539680481, + "0.7": 0.895737886428833, + "0.8": 0.895737886428833, + "0.9": 0.895737886428833 + }, + "unique_frac_signed": { + "0.5": 0.8197428385416666, + "0.6": 0.999267578125, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7338954136988225, + "0.6": 0.894824584494927, + "0.7": 0.895737886428833, + "0.8": 0.895737886428833, + "0.9": 0.895737886428833 + }, + "matched_frac_signed": { + "0.5": 0.7716471354166666, + "0.6": 0.9358723958333334, + "0.7": 0.90576171875, + "0.8": 0.8473307291666666, + "0.9": 0.72607421875 + }, + "matched_cos_abs": { + "0.5": 0.8959991335868835, + "0.6": 0.8965321183204651, + "0.7": 0.8961375951766968, + "0.8": 0.8958845734596252, + "0.9": 0.8958330750465393 + }, + "unique_frac_abs": { + "0.5": 0.8185221354166666, + "0.6": 0.9921875, + "0.7": 0.99755859375, + "0.8": 0.9999186197916666, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.7333951241550191, + "0.6": 0.8895279611460865, + "0.7": 0.8939497592509724, + "0.8": 0.8958116661863944, + "0.9": 0.8958330750465393 + }, + "matched_frac_abs": { + "0.5": 0.7706705729166666, + "0.6": 0.9285481770833334, + "0.7": 0.9036458333333334, + "0.8": 0.8473307291666666, + "0.9": 0.7259114583333334 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8961827158927917, + "dcs_abs": 0.8962742686271667, + "frac_sign_flipped": 0.0029296875, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8954179286956787, + "0.6": 0.8961955308914185, + "0.7": 0.8961827158927917, + "0.8": 0.8961827158927917, + "0.9": 0.8961827158927917 + }, + "unique_frac_signed": { + "0.5": 0.8214518229166666, + "0.6": 0.99951171875, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7355426897993311, + "0.6": 0.8957579354173504, + "0.7": 0.8961827158927917, + "0.8": 0.8961827158927917, + "0.9": 0.8961827158927917 + }, + "matched_frac_signed": { + "0.5": 0.7740885416666666, + "0.6": 0.9348958333333334, + "0.7": 0.9073893229166666, + "0.8": 0.8450520833333334, + "0.9": 0.7224934895833334 + }, + "matched_cos_abs": { + "0.5": 0.8954559564590454, + "0.6": 0.8969697952270508, + "0.7": 0.896643877029419, + "0.8": 0.8963789343833923, + "0.9": 0.8962742686271667 + }, + "unique_frac_abs": { + "0.5": 0.8197428385416666, + "0.6": 0.992431640625, + "0.7": 0.99755859375, + "0.8": 0.9998372395833334, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.7340436075367809, + "0.6": 0.8901812054682523, + "0.7": 0.8944548050640151, + "0.8": 0.8962330393745409, + "0.9": 0.8962742686271667 + }, + "matched_frac_abs": { + "0.5": 0.7721354166666666, + "0.6": 0.9278971354166666, + "0.7": 0.9054361979166666, + "0.8": 0.8450520833333334, + "0.9": 0.7224934895833334 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": 37, + "l0_std": 0.2440429152723829, + "recon_r2_mean": 0.8995203111076936, + "recon_r2_std": 9.01144933050647e-05, + "l1_per_token_mean": 201.1068835874818, + "l1_per_token_std": 0.17013063738594808, + "babel_mu1_signed_mean": 0.4538206656773885, + "babel_mu1_signed_std": 3.978763326151768e-05, + "n_dup_signed_per_threshold": { + "0.5": 2012.6666666666667, + "0.6": 6.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 1.5, + "n_dup_signed_band_std": 0.4330127018922193, + "n_dup_clusters_signed_per_threshold": { + "0.5": 910.3333333333334, + "0.6": 2.3333333333333335, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.5833333333333334, + "n_dup_clusters_signed_band_std": 0.14433756729740646, + "largest_dup_signed_per_threshold": { + "0.5": 5.666666666666667, + "0.6": 3.3333333333333335, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.5833333333333335, + "largest_dup_signed_band_std": 0.38188130791298663, + "dcs_signed_mean": 0.8962082664171854, + "dcs_signed_std": 0.0004836616773179039, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7349197475672957, + "0.6": 0.895556581350522, + "0.7": 0.8962082664171854, + "0.8": 0.8962082664171854, + "0.9": 0.8962082664171854 + }, + "dedup_dcs_signed_band": 0.8960453451505195, + "dedup_dcs_signed_band_std": 0.0005208965611205338, + "matched_cos_signed_per_threshold": { + "0.5": 0.8956059614817301, + "0.6": 0.8960913221041361, + "0.7": 0.8962082664171854, + "0.8": 0.8962082664171854, + "0.9": 0.8962082664171854 + }, + "matched_cos_signed_band": 0.8961790303389231, + "matched_cos_signed_band_std": 0.0005020928232775548, + "unique_frac_signed_per_threshold": { + "0.5": 0.820583767361111, + "0.6": 0.9994032118055555, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9998508029513888, + "unique_frac_signed_band_std": 3.107758039656063e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.7734917534722222, + "0.6": 0.9350043402777778, + "0.7": 0.9063042534722222, + "0.8": 0.84716796875, + "0.9": 0.7263454861111112 + }, + "matched_frac_signed_band": 0.8537055121527778, + "matched_frac_signed_band_std": 0.0012216070148868476, + "babel_mu1_abs_mean": 0.45815162857373554, + "babel_mu1_abs_std": 0.00010725408796981306, + "n_dup_abs_per_threshold": { + "0.5": 2028.6666666666667, + "0.6": 94.0, + "0.7": 29.333333333333332, + "0.8": 2.0, + "0.9": 0.0 + }, + "n_dup_abs_band": 31.333333333333332, + "n_dup_abs_band_std": 0.5204164998665333, + "n_dup_clusters_abs_per_threshold": { + "0.5": 916.6666666666666, + "0.6": 46.333333333333336, + "0.7": 14.666666666666666, + "0.8": 1.0, + "0.9": 0.0 + }, + "n_dup_clusters_abs_band": 15.5, + "n_dup_clusters_abs_band_std": 0.25, + "largest_dup_abs_per_threshold": { + "0.5": 5.666666666666667, + "0.6": 3.3333333333333335, + "0.7": 2.0, + "0.8": 1.6666666666666667, + "0.9": 1.0 + }, + "largest_dup_abs_band": 2.0, + "largest_dup_abs_band_std": 0.4330127018922193, + "dcs_abs_mean": 0.896316925684611, + "dcs_abs_std": 0.0005065280929769378, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.7338272250732795, + "0.6": 0.8901133105844362, + "0.7": 0.8944532868853355, + "0.8": 0.8962556766523954, + "0.9": 0.896316925684611 + }, + "dedup_dcs_abs_band": 0.8942847999516945, + "dedup_dcs_abs_band_std": 0.000503670492686034, + "matched_cos_abs_per_threshold": { + "0.5": 0.8959927558898926, + "0.6": 0.8970730503400167, + "0.7": 0.8965935707092285, + "0.8": 0.896401604016622, + "0.9": 0.896316925684611 + }, + "matched_cos_abs_band": 0.8965962876876196, + "matched_cos_abs_band_std": 0.0005148688181305283, + "unique_frac_abs_per_threshold": { + "0.5": 0.8190104166666666, + "0.6": 0.9922417534722222, + "0.7": 0.9976128472222222, + "0.8": 0.9998372395833334, + "0.9": 1.0 + }, + "unique_frac_abs_band": 0.9974229600694444, + "unique_frac_abs_band_std": 3.10775803966212e-05, + "matched_frac_abs_per_threshold": { + "0.5": 0.7721896701388888, + "0.6": 0.9280056423611112, + "0.7": 0.9042426215277778, + "0.8": 0.8472764756944445, + "0.9": 0.726291232638889 + }, + "matched_frac_abs_band": 0.8514539930555556, + "matched_frac_abs_band_std": 0.0012843728950911537, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 6144 + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_32/seed0/config.json b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8bc1106ed568ec38ba537b5ca71504f290a48c69 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed0/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 8.38, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 15, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..0b36e0cf68d84b775c566bc92620210202242b8e --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 36.65106376953125, + "l0_per_token_0.001": 36.65103618164063, + "l0_per_token_0.01": 36.65094716796875, + "l0_per_token_0.05": 36.65063037109375, + "l0_per_token_0.1": 36.6501505859375, + "l0_per_token_0.5": 36.63534091796875, + "l0_per_token_1.0": 36.434508837890625, + "l1_per_token": 201.06601714415362, + "r2_score": 0.8994488106656073, + "l2_loss": 2.358170133046022 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_32/seed1/config.json b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3fede86a0b22fc3feb9bab4647ebf5fe4c7740da --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed1/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 8.38, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 15, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..253dbfadcf6d18f88ce752c2173704de975e2adf --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 36.2884943359375, + "l0_per_token_0.001": 36.288486376953124, + "l0_per_token_0.01": 36.28840854492187, + "l0_per_token_0.05": 36.28806489257813, + "l0_per_token_0.1": 36.28733881835937, + "l0_per_token_0.5": 36.27286787109375, + "l0_per_token_1.0": 36.112089990234374, + "l1_per_token": 201.2937255953762, + "r2_score": 0.8996215318334677, + "l2_loss": 2.354119401253696 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_32/seed2/config.json b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..54f4a484bcaea21d4146b18f912f617ec3cac378 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed2/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 8.38, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 15, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..525eeff5bce9699a5e33833dbd3a96ecc66f7971 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 36.75276030273437, + "l0_per_token_0.001": 36.75275170898438, + "l0_per_token_0.01": 36.75265947265625, + "l0_per_token_0.05": 36.752208984375, + "l0_per_token_0.1": 36.7515595703125, + "l0_per_token_0.5": 36.7390845703125, + "l0_per_token_1.0": 36.53483530273437, + "l1_per_token": 200.9609080229157, + "r2_score": 0.8994905908240057, + "l2_loss": 2.3571902866380277 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_64/code_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x8/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..3c5c7c62485cde87f63b0dc8039d4b1e3634358c --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 64.00388, + "recon_r2": 0.9222036158869706, + "l1_per_token": 252.1489402350096 + }, + "seed1": { + "l0": 64.00326333333334, + "recon_r2": 0.9222111271404687, + "l1_per_token": 252.04342295441742 + }, + "seed2": { + "l0": 63.561395, + "recon_r2": 0.9223384459955895, + "l1_per_token": 252.2637088622582 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.8362828531323844, + "mcc_abs": 0.8362864434550211, + "act_iou": 0.5338078206262785, + "act_iou_n_pairs": 6144, + "input_r2": 0.9888768604088013, + "matched_mcc_signed": { + "0.6": 0.8890783005091274, + "0.7": 0.9039807951267466, + "0.8": 0.92578483452708, + "0.9": 0.9507428206259048 + }, + "matched_mcc_abs": { + "0.6": 0.8895495471139431, + "0.7": 0.9038450952662357, + "0.8": 0.9255284851417173, + "0.9": 0.9507485687747284 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.8434882731846982, + "mcc_abs": 0.843491480588046, + "act_iou": 0.5416336601514246, + "act_iou_n_pairs": 6144, + "input_r2": 0.9891664321791883, + "matched_mcc_signed": { + "0.6": 0.8901458159131806, + "0.7": 0.9078755536560317, + "0.8": 0.927411025714398, + "0.9": 0.9502554915898485 + }, + "matched_mcc_abs": { + "0.6": 0.8911167503755235, + "0.7": 0.9073932116199237, + "0.8": 0.9273532487306478, + "0.9": 0.9502727246343017 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.8424063760254037, + "mcc_abs": 0.8424094915996161, + "act_iou": 0.5432839505587629, + "act_iou_n_pairs": 6144, + "input_r2": 0.9891444101660765, + "matched_mcc_signed": { + "0.6": 0.8901679319388017, + "0.7": 0.9069060406065619, + "0.8": 0.9276973054348396, + "0.9": 0.9523446035210255 + }, + "matched_mcc_abs": { + "0.6": 0.8905271623673827, + "0.7": 0.9057045694687278, + "0.8": 0.9278269501617253, + "0.9": 0.952344651392431 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 63.85617944444444, + "l0_std": 0.25529100372762803, + "recon_r2_mean": 0.9222510630076762, + "recon_r2_std": 7.576902180149048e-05, + "l1_per_token_mean": 252.15202401722843, + "l1_per_token_std": 0.11017532656129693, + "input_r2_mean": 0.9890625675846887, + "input_r2_std": 0.00016120362479618325, + "mcc_signed_mean": 0.8407258341141621, + "mcc_signed_std": 0.0038855740076790793, + "mcc_abs_mean": 0.840729138547561, + "mcc_abs_std": 0.0038853352244986334, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8897973494537031, + "0.7": 0.9062541297964467, + "0.8": 0.9269643885587726, + "0.9": 0.9511143052455928 + }, + "matched_mcc_signed_band": 0.9185325432636289, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8903978199522831, + "0.7": 0.9056476254516291, + "0.8": 0.9269028946780301, + "0.9": 0.9511219816004871 + }, + "matched_mcc_abs_band": 0.9185175804206074, + "act_iou_mean": 0.539575143778822, + "act_iou_std": 0.005062348947527835, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_64/dict_metrics_step_final.json b/lm/gpt2-small_layer8/consdl/x8/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..61890ae5d8b62b6615b1db3acabc56adca96165d --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_64/dict_metrics_step_final.json @@ -0,0 +1,550 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 64.01294296875, + "recon_r2": 0.9410412023435403, + "l1_per_token": 252.08988293467792, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 8, + "dense_rate": 0.0013020833333333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1394, + "0.6": 5, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 637, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 5, + "0.6": 5, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.4253605604171753, + "n_dup_abs": { + "0.5": 1437, + "0.6": 103, + "0.7": 34, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 661, + "0.6": 50, + "0.7": 17, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 5, + "0.6": 5, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.43104490637779236 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 64.04282661132812, + "recon_r2": 0.9410448813452562, + "l1_per_token": 252.01399579184698, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 6, + "dense_rate": 0.0009765625, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1376, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 623, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.4249289929866791, + "n_dup_abs": { + "0.5": 1413, + "0.6": 94, + "0.7": 38, + "0.8": 6, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 644, + "0.6": 47, + "0.7": 19, + "0.8": 3, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_abs": 0.43058082461357117 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 63.5786166015625, + "recon_r2": 0.9222040066621313, + "l1_per_token": 252.219439814311, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 9, + "dense_rate": 0.00146484375, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1413, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 643, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.4254709780216217, + "n_dup_abs": { + "0.5": 1443, + "0.6": 104, + "0.7": 32, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_abs": { + "0.5": 657, + "0.6": 52, + "0.7": 16, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_abs": 0.43116316199302673 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8880100846290588, + "dcs_abs": 0.8881585001945496, + "frac_sign_flipped": 0.0037434895833333335, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.885786771774292, + "0.6": 0.8880087733268738, + "0.7": 0.8880100846290588, + "0.8": 0.8880100846290588, + "0.9": 0.8880100846290588 + }, + "unique_frac_signed": { + "0.5": 0.8771158854166666, + "0.6": 0.9996744791666666, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7769376486151789, + "0.6": 0.8877197079709731, + "0.7": 0.8880100846290588, + "0.8": 0.8880100846290588, + "0.9": 0.8880100846290588 + }, + "matched_frac_signed": { + "0.5": 0.82177734375, + "0.6": 0.9220377604166666, + "0.7": 0.8907877604166666, + "0.8": 0.8304036458333334, + "0.9": 0.7169596354166666 + }, + "matched_cos_abs": { + "0.5": 0.8868375420570374, + "0.6": 0.8892170190811157, + "0.7": 0.8886439800262451, + "0.8": 0.8882742524147034, + "0.9": 0.8881585001945496 + }, + "unique_frac_abs": { + "0.5": 0.874267578125, + "0.6": 0.9918619791666666, + "0.7": 0.9970703125, + "0.8": 0.9996744791666666, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.7753333100845339, + "0.6": 0.881980552454479, + "0.7": 0.886040530866012, + "0.8": 0.8879851006398287, + "0.9": 0.8881585001945496 + }, + "matched_frac_abs": { + "0.5": 0.81982421875, + "0.6": 0.9140625, + "0.7": 0.8878580729166666, + "0.8": 0.83056640625, + "0.9": 0.716796875 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8931569457054138, + "dcs_abs": 0.8933312296867371, + "frac_sign_flipped": 0.004069010416666667, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.89134681224823, + "0.6": 0.8932076692581177, + "0.7": 0.8931570053100586, + "0.8": 0.8931570053100586, + "0.9": 0.8931570053100586 + }, + "unique_frac_signed": { + "0.5": 0.875732421875, + "0.6": 0.9996744791666666, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7805813026207034, + "0.6": 0.892916911553281, + "0.7": 0.8931570053100586, + "0.8": 0.8931570053100586, + "0.9": 0.8931570053100586 + }, + "matched_frac_signed": { + "0.5": 0.8229166666666666, + "0.6": 0.9313151041666666, + "0.7": 0.8958333333333334, + "0.8": 0.8395182291666666, + "0.9": 0.7283528645833334 + }, + "matched_cos_abs": { + "0.5": 0.8918891549110413, + "0.6": 0.8943801522254944, + "0.7": 0.89378821849823, + "0.8": 0.8934060335159302, + "0.9": 0.8933311104774475 + }, + "unique_frac_abs": { + "0.5": 0.8728841145833334, + "0.6": 0.991455078125, + "0.7": 0.997314453125, + "0.8": 0.9999186197916666, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.7785158752910017, + "0.6": 0.8867377436981769, + "0.7": 0.8913879083411302, + "0.8": 0.8933333279467963, + "0.9": 0.8933311104774475 + }, + "matched_frac_abs": { + "0.5": 0.8201497395833334, + "0.6": 0.9228515625, + "0.7": 0.8937174479166666, + "0.8": 0.8391927083333334, + "0.9": 0.7281901041666666 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8929831385612488, + "dcs_abs": 0.8932056427001953, + "frac_sign_flipped": 0.005533854166666667, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8926342129707336, + "0.6": 0.8929831385612488, + "0.7": 0.8929831385612488, + "0.8": 0.8929831385612488, + "0.9": 0.8929831385612488 + }, + "unique_frac_signed": { + "0.5": 0.8760579427083334, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7819992922062132, + "0.6": 0.8929831385612488, + "0.7": 0.8929831385612488, + "0.8": 0.8929831385612488, + "0.9": 0.8929831385612488 + }, + "matched_frac_signed": { + "0.5": 0.8243815104166666, + "0.6": 0.9308268229166666, + "0.7": 0.8951822916666666, + "0.8": 0.8375651041666666, + "0.9": 0.72900390625 + }, + "matched_cos_abs": { + "0.5": 0.8932338953018188, + "0.6": 0.8946171998977661, + "0.7": 0.8937267065048218, + "0.8": 0.8933151960372925, + "0.9": 0.8932056427001953 + }, + "unique_frac_abs": { + "0.5": 0.8734537760416666, + "0.6": 0.991943359375, + "0.7": 0.9971516927083334, + "0.8": 0.999755859375, + "0.9": 1.0 + }, + "dedup_dcs_abs": { + "0.5": 0.7801985187397804, + "0.6": 0.887409590621246, + "0.7": 0.8911810982099269, + "0.8": 0.8930971015070099, + "0.9": 0.8932056427001953 + }, + "matched_frac_abs": { + "0.5": 0.8229166666666666, + "0.6": 0.9226888020833334, + "0.7": 0.89404296875, + "0.8": 0.8370768229166666, + "0.9": 0.72900390625 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 8, + "n_dense_std": 1.5275252316519465, + "l0_mean": 64, + "l0_std": 0.25981511326794915, + "recon_r2_mean": 0.9347633634503092, + "recon_r2_std": 0.010876722189305339, + "l1_per_token_mean": 252.1077728469453, + "l1_per_token_std": 0.1038838212247161, + "babel_mu1_signed_mean": 0.4252535104751587, + "babel_mu1_signed_std": 0.00028641178762362147, + "n_dup_signed_per_threshold": { + "0.5": 1394.3333333333333, + "0.6": 1.6666666666666667, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.4166666666666667, + "n_dup_signed_band_std": 0.7216878364870322, + "n_dup_clusters_signed_per_threshold": { + "0.5": 634.3333333333334, + "0.6": 0.3333333333333333, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.08333333333333333, + "n_dup_clusters_signed_band_std": 0.14433756729740646, + "largest_dup_signed_per_threshold": { + "0.5": 5.666666666666667, + "0.6": 2.3333333333333335, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.3333333333333335, + "largest_dup_signed_band_std": 0.5773502691896257, + "dcs_signed_mean": 0.8913833896319071, + "dcs_signed_std": 0.0029226601260703864, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7798394144806985, + "0.6": 0.8912065860285009, + "0.7": 0.8913834095001221, + "0.8": 0.8913834095001221, + "0.9": 0.8913834095001221 + }, + "dedup_dcs_signed_band": 0.8913392036322167, + "dedup_dcs_signed_band_std": 0.0029465198847383515, + "matched_cos_signed_per_threshold": { + "0.5": 0.8899225989977518, + "0.6": 0.8913998603820801, + "0.7": 0.8913834095001221, + "0.8": 0.8913834095001221, + "0.9": 0.8913834095001221 + }, + "matched_cos_signed_band": 0.8913875222206116, + "matched_cos_signed_band_std": 0.0029267171071178082, + "unique_frac_signed_per_threshold": { + "0.5": 0.8763020833333334, + "0.6": 0.999782986111111, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9999457465277778, + "unique_frac_signed_band_std": 4.698488518797919e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.823025173611111, + "0.6": 0.9280598958333334, + "0.7": 0.8939344618055555, + "0.8": 0.8358289930555555, + "0.9": 0.7247721354166666 + }, + "matched_frac_signed_band": 0.8456488715277777, + "matched_frac_signed_band_std": 0.004860778827512464, + "babel_mu1_abs_mean": 0.43092963099479675, + "babel_mu1_abs_std": 0.000307807596269783, + "n_dup_abs_per_threshold": { + "0.5": 1431.0, + "0.6": 100.33333333333333, + "0.7": 34.666666666666664, + "0.8": 2.6666666666666665, + "0.9": 0.0 + }, + "n_dup_abs_band": 34.416666666666664, + "n_dup_abs_band_std": 0.38188130791298663, + "n_dup_clusters_abs_per_threshold": { + "0.5": 654.0, + "0.6": 49.666666666666664, + "0.7": 17.333333333333332, + "0.8": 1.3333333333333333, + "0.9": 0.0 + }, + "n_dup_clusters_abs_band": 17.083333333333332, + "n_dup_clusters_abs_band_std": 0.14433756729740643, + "largest_dup_abs_per_threshold": { + "0.5": 5.666666666666667, + "0.6": 3.0, + "0.7": 2.0, + "0.8": 1.6666666666666667, + "0.9": 1.0 + }, + "largest_dup_abs_band": 1.9166666666666667, + "largest_dup_abs_band_std": 0.5204164998665332, + "dcs_abs_mean": 0.8915651241938273, + "dcs_abs_std": 0.002950891107299372, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.7780159013717721, + "0.6": 0.8853759622579673, + "0.7": 0.8895365124723563, + "0.8": 0.891471843364545, + "0.9": 0.8915650844573975 + }, + "dedup_dcs_abs_band": 0.8894873506380665, + "dedup_dcs_abs_band_std": 0.002984507035232042, + "matched_cos_abs_per_threshold": { + "0.5": 0.8906535307566324, + "0.6": 0.8927381237347921, + "0.7": 0.892052968343099, + "0.8": 0.8916651606559753, + "0.9": 0.8915650844573975 + }, + "matched_cos_abs_band": 0.892005334297816, + "matched_cos_abs_band_std": 0.002972113807571035, + "unique_frac_abs_per_threshold": { + "0.5": 0.87353515625, + "0.6": 0.9917534722222222, + "0.7": 0.9971788194444445, + "0.8": 0.999782986111111, + "0.9": 1.0 + }, + "unique_frac_abs_band": 0.9971788194444444, + "unique_frac_abs_band_std": 3.10775803966212e-05, + "matched_frac_abs_per_threshold": { + "0.5": 0.8209635416666666, + "0.6": 0.9198676215277778, + "0.7": 0.891872829861111, + "0.8": 0.8356119791666666, + "0.9": 0.7246636284722222 + }, + "matched_frac_abs_band": 0.8430040147569444, + "matched_frac_abs_band_std": 0.004923726786964542, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 6144 + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_64/seed0/config.json b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..44358aea843fb51c17beee09d4d9895b896db240 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed0/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 5.35, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 10, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..388b1594f0edf9790a6dbf70fbc37ae9588a03dd --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.01294296875, + "l0_per_token_0.001": 64.01283544921876, + "l0_per_token_0.01": 64.01255883789062, + "l0_per_token_0.05": 64.0116646484375, + "l0_per_token_0.1": 64.01024096679687, + "l0_per_token_0.5": 63.941675537109376, + "l0_per_token_1.0": 54.46874638671875, + "l1_per_token": 252.08988293467792, + "r2_score": 0.9410412023435403, + "l2_loss": 1.827734617078848 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_64/seed1/config.json b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fe714a999d290bd89ff6644fa1d8062022f4bc94 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed1/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 5.35, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 10, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e56da6be03ed126d7e1eb495a40fa745b3ff8b3e --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.04282661132812, + "l0_per_token_0.001": 64.04278081054687, + "l0_per_token_0.01": 64.0424828125, + "l0_per_token_0.05": 64.04096201171875, + "l0_per_token_0.1": 64.03860786132813, + "l0_per_token_0.5": 63.96262294921875, + "l0_per_token_1.0": 54.44533520507812, + "l1_per_token": 252.01399579184698, + "r2_score": 0.9410448813452562, + "l2_loss": 1.8276205672837416 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_64/seed2/config.json b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b8c06c346c52e9ebc0a1c0e7774b151c2dd5dcf5 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed2/config.json @@ -0,0 +1,24 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "fista", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "fista_t_iters": 30, + "fista_step": 0.0, + "fista_lam": 5.35, + "fista_nonneg": true, + "fista_power_iters": 32, + "fista_r2_tol": 0.0001, + "fista_check_every": 3, + "fista_use_bf16": false, + "fista_step_cache_every": 50, + "fista_compile": false, + "fista_topk_k": 0, + "refit": true, + "refit_iters": 10, + "refit_thresh": 1e-08, + "class_type": "FistaSAE", + "module_name": "consdl.models.fista" +} diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..9239b75667d770c1b13b79ec5d7a2e247c1396ae --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.5786166015625, + "l0_per_token_0.001": 63.578586474609374, + "l0_per_token_0.01": 63.57840170898437, + "l0_per_token_0.05": 63.577488330078126, + "l0_per_token_0.1": 63.57609594726563, + "l0_per_token_0.5": 63.51887006835938, + "l0_per_token_1.0": 54.37858481445313, + "l1_per_token": 252.219439814311, + "r2_score": 0.9222040066621313, + "l2_loss": 1.8245054004275194 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/consdl/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c933ddf2467a7fe8b200254aaf4a7548573f1371 --- /dev/null +++ b/lm/gpt2-small_layer8/consdl/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/topk/x16/l0_128/code_metrics_step_final.json b/lm/gpt2-small_layer8/topk/x16/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..d7b0d4a1cb175281e4827c061bb31afb1e4b352e --- /dev/null +++ b/lm/gpt2-small_layer8/topk/x16/l0_128/code_metrics_step_final.json @@ -0,0 +1,87 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "dedup_mcc": false, + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 127.997045, + "recon_r2": 0.9535773371234301, + "l1_per_token": 589.7256349381423 + }, + "seed1": { + "l0": 127.99953833333333, + "recon_r2": 0.9536457882715428, + "l1_per_token": 589.4080771051748 + }, + "seed2": { + "l0": 127.99587166666667, + "recon_r2": 0.9535653847375515, + "l1_per_token": 588.9491381925291 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6705226594468109, + "mcc_abs": 0.6705582007488969, + "act_iou": 0.3349847350131427, + "act_iou_n_pairs": 12288, + "input_r2": 0.9712833186683074 + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6701303602826639, + "mcc_abs": 0.6701632722225223, + "act_iou": 0.33499746586913637, + "act_iou_n_pairs": 12288, + "input_r2": 0.9710728975429641 + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6696030774105646, + "mcc_abs": 0.669624917677789, + "act_iou": 0.334429003926265, + "act_iou_n_pairs": 12288, + "input_r2": 0.9714008694046359 + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 127.997485, + "l0_std": 0.001872514649101923, + "recon_r2_mean": 0.9535961700441747, + "recon_r2_std": 4.338422809506901e-05, + "l1_per_token_mean": 589.3609500786155, + "l1_per_token_std": 0.3903876489609321, + "input_r2_mean": 0.9712523618719692, + "input_r2_std": 0.00016616295598525965, + "mcc_signed_mean": 0.6700853657133464, + "mcc_signed_std": 0.0004614392308828729, + "mcc_abs_mean": 0.6701154635497361, + "mcc_abs_std": 0.00046847473208206183, + "act_iou_mean": 0.3348037349361814, + "act_iou_std": 0.0003245889955286151, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/topk/x16/l0_128/dict_metrics_step_final.json b/lm/gpt2-small_layer8/topk/x16/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..7976ab5c2a02f5c88ae24460a5a862c990612e74 --- /dev/null +++ b/lm/gpt2-small_layer8/topk/x16/l0_128/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 128, + "l0_raw": 127.99712387695313, + "recon_r2": 0.9535042271919859, + "l1_per_token": 589.6484688185383, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 216, + "dense_rate": 0.017578125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 289, + "0.6": 90, + "0.7": 39, + "0.8": 19, + "0.9": 12 + }, + "n_dup_clusters_signed": { + "0.5": 137, + "0.6": 38, + "0.7": 14, + "0.8": 5, + "0.9": 2 + }, + "n_dup_removed_signed": { + "0.5": 152, + "0.6": 52, + "0.7": 25, + "0.8": 14, + "0.9": 10 + }, + "largest_dup_signed": { + "0.5": 11, + "0.6": 11, + "0.7": 11, + "0.8": 11, + "0.9": 10 + }, + "babel_mu1_signed": 0.28956660628318787, + "n_dup_abs": { + "0.5": 912, + "0.6": 634, + "0.7": 540, + "0.8": 464, + "0.9": 347 + }, + "n_dup_clusters_abs": { + "0.5": 443, + "0.6": 307, + "0.7": 263, + "0.8": 227, + "0.9": 169 + }, + "n_dup_removed_abs": { + "0.5": 469, + "0.6": 327, + "0.7": 277, + "0.8": 237, + "0.9": 178 + }, + "largest_dup_abs": { + "0.5": 11, + "0.6": 11, + "0.7": 11, + "0.8": 11, + "0.9": 10 + }, + "babel_mu1_abs": 0.33405303955078125, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 128, + "l0_raw": 127.99949423828124, + "recon_r2": 0.9535554051583998, + "l1_per_token": 589.2194232140833, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 215, + "dense_rate": 0.017496744791666668, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 256, + "0.6": 67, + "0.7": 17, + "0.8": 9, + "0.9": 6 + }, + "n_dup_clusters_signed": { + "0.5": 124, + "0.6": 31, + "0.7": 6, + "0.8": 2, + "0.9": 1 + }, + "n_dup_removed_signed": { + "0.5": 132, + "0.6": 36, + "0.7": 11, + "0.8": 7, + "0.9": 5 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 6, + "0.7": 6, + "0.8": 6, + "0.9": 6 + }, + "babel_mu1_signed": 0.28825643658638, + "n_dup_abs": { + "0.5": 874, + "0.6": 631, + "0.7": 533, + "0.8": 453, + "0.9": 342 + }, + "n_dup_clusters_abs": { + "0.5": 430, + "0.6": 311, + "0.7": 264, + "0.8": 224, + "0.9": 169 + }, + "n_dup_removed_abs": { + "0.5": 444, + "0.6": 320, + "0.7": 269, + "0.8": 229, + "0.9": 173 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 6, + "0.7": 6, + "0.8": 6, + "0.9": 6 + }, + "babel_mu1_abs": 0.332329660654068, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 128, + "l0_raw": 127.99571987304688, + "recon_r2": 0.9534961410314295, + "l1_per_token": 588.8183802243086, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 222, + "dense_rate": 0.01806640625, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 268, + "0.6": 94, + "0.7": 33, + "0.8": 13, + "0.9": 6 + }, + "n_dup_clusters_signed": { + "0.5": 128, + "0.6": 43, + "0.7": 14, + "0.8": 4, + "0.9": 1 + }, + "n_dup_removed_signed": { + "0.5": 140, + "0.6": 51, + "0.7": 19, + "0.8": 9, + "0.9": 5 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 6, + "0.7": 6, + "0.8": 6, + "0.9": 6 + }, + "babel_mu1_signed": 0.2906745374202728, + "n_dup_abs": { + "0.5": 899, + "0.6": 662, + "0.7": 547, + "0.8": 476, + "0.9": 352 + }, + "n_dup_clusters_abs": { + "0.5": 441, + "0.6": 326, + "0.7": 270, + "0.8": 235, + "0.9": 174 + }, + "n_dup_removed_abs": { + "0.5": 458, + "0.6": 336, + "0.7": 277, + "0.8": 241, + "0.9": 178 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 6, + "0.7": 6, + "0.8": 6, + "0.9": 6 + }, + "babel_mu1_abs": 0.3354865610599518, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.6879796981811523, + "dcs_abs": 0.6890723705291748, + "frac_sign_flipped": 0.024251302083333332, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.6861667037010193, + "0.6": 0.6878974437713623, + "0.7": 0.6884343028068542, + "0.8": 0.68818598985672, + "0.9": 0.6881309151649475 + }, + "unique_frac_signed": { + "0.5": 0.9884440104166666, + "0.6": 0.9964192708333334, + "0.7": 0.99853515625, + "0.8": 0.9991455078125, + "0.9": 0.9993896484375 + }, + "dedup_dcs_signed": { + "0.5": 0.6782373684206201, + "0.6": 0.6854342693307748, + "0.7": 0.687425854121102, + "0.8": 0.6875979403048404, + "0.9": 0.687710913385672 + }, + "matched_frac_signed": { + "0.5": 0.7303873697916666, + "0.6": 0.6642252604166666, + "0.7": 0.57568359375, + "0.8": 0.458251953125, + "0.9": 0.2649739583333333 + }, + "matched_cos_abs": { + "0.5": 0.6895968317985535, + "0.6": 0.690534770488739, + "0.7": 0.6904430985450745, + "0.8": 0.6901004910469055, + "0.9": 0.6899100542068481 + }, + "unique_frac_abs": { + "0.5": 0.9628499348958334, + "0.6": 0.9736735026041666, + "0.7": 0.977783203125, + "0.8": 0.9810384114583334, + "0.9": 0.9857177734375 + }, + "dedup_dcs_abs": { + "0.5": 0.6639782646016101, + "0.6": 0.6723554086517348, + "0.7": 0.6751036644709529, + "0.8": 0.677015089483272, + "0.9": 0.6800566025049193 + }, + "matched_frac_abs": { + "0.5": 0.7152506510416666, + "0.6": 0.655029296875, + "0.7": 0.5680338541666666, + "0.8": 0.4518229166666667, + "0.9": 0.2618001302083333 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.6869797706604004, + "dcs_abs": 0.6881580352783203, + "frac_sign_flipped": 0.024495442708333332, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.6846131086349487, + "0.6": 0.68647700548172, + "0.7": 0.6871971487998962, + "0.8": 0.6871720552444458, + "0.9": 0.6871265172958374 + }, + "unique_frac_signed": { + "0.5": 0.9881184895833334, + "0.6": 0.9958089192708334, + "0.7": 0.9982096354166666, + "0.8": 0.9990641276041666, + "0.9": 0.9993896484375 + }, + "dedup_dcs_signed": { + "0.5": 0.676478870853316, + "0.6": 0.6835999249330296, + "0.7": 0.6859668153629173, + "0.8": 0.6865289498867544, + "0.9": 0.6867071285523707 + }, + "matched_frac_signed": { + "0.5": 0.730224609375, + "0.6": 0.6605631510416666, + "0.7": 0.57666015625, + "0.8": 0.4583333333333333, + "0.9": 0.25732421875 + }, + "matched_cos_abs": { + "0.5": 0.6881800293922424, + "0.6": 0.6900156140327454, + "0.7": 0.6898819208145142, + "0.8": 0.6895862817764282, + "0.9": 0.6891046166419983 + }, + "unique_frac_abs": { + "0.5": 0.9622802734375, + "0.6": 0.9730224609375, + "0.7": 0.9774576822916666, + "0.8": 0.9805501302083334, + "0.9": 0.9855143229166666 + }, + "dedup_dcs_abs": { + "0.5": 0.6622220668577938, + "0.6": 0.671400690851442, + "0.7": 0.6743303833742781, + "0.8": 0.6761739183857571, + "0.9": 0.6791224696886881 + }, + "matched_frac_abs": { + "0.5": 0.7159830729166666, + "0.6": 0.6507975260416666, + "0.7": 0.5696614583333334, + "0.8": 0.4525553385416667, + "0.9": 0.2542317708333333 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.686482846736908, + "dcs_abs": 0.687578022480011, + "frac_sign_flipped": 0.023274739583333332, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.6839383244514465, + "0.6": 0.6864351034164429, + "0.7": 0.6865894198417664, + "0.8": 0.6864301562309265, + "0.9": 0.686367392539978 + }, + "unique_frac_signed": { + "0.5": 0.9889322916666666, + "0.6": 0.9964599609375, + "0.7": 0.998779296875, + "0.8": 0.9993489583333334, + "0.9": 0.9995930989583334 + }, + "dedup_dcs_signed": { + "0.5": 0.6763686945584292, + "0.6": 0.6840050963364774, + "0.7": 0.6857512979913736, + "0.8": 0.6859832615979637, + "0.9": 0.6860881089329876 + }, + "matched_frac_signed": { + "0.5": 0.7311197916666666, + "0.6": 0.6608072916666666, + "0.7": 0.5673828125, + "0.8": 0.453125, + "0.9": 0.2611490885416667 + }, + "matched_cos_abs": { + "0.5": 0.6877094507217407, + "0.6": 0.6895751357078552, + "0.7": 0.6892557144165039, + "0.8": 0.688835859298706, + "0.9": 0.6881998777389526 + }, + "unique_frac_abs": { + "0.5": 0.9632975260416666, + "0.6": 0.9733072916666666, + "0.7": 0.977783203125, + "0.8": 0.9808756510416666, + "0.9": 0.9857177734375 + }, + "dedup_dcs_abs": { + "0.5": 0.6624688125157263, + "0.6": 0.6711685077364866, + "0.7": 0.6739426602143794, + "0.8": 0.6756623219504642, + "0.9": 0.6783708511648001 + }, + "matched_frac_abs": { + "0.5": 0.71826171875, + "0.6": 0.6520182291666666, + "0.7": 0.5592447916666666, + "0.8": 0.447021484375, + "0.9": 0.2579752604166667 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 218, + "n_dense_std": 3.7859388972001824, + "l0_mean": 128, + "l0_std": 0.0019076893840876228, + "recon_r2_mean": 0.9535185911272718, + "recon_r2_std": 3.2137223963140725e-05, + "l1_per_token_mean": 589.2287574189767, + "l1_per_token_std": 0.41512301080919267, + "babel_mu1_signed_mean": 0.2894991934299469, + "babel_mu1_signed_std": 0.0012104591196338919, + "n_dup_signed_per_threshold": { + "0.5": 271.0, + "0.6": 83.66666666666667, + "0.7": 29.666666666666668, + "0.8": 13.666666666666666, + "0.9": 8.0 + }, + "n_dup_signed_band": 33.75, + "n_dup_signed_band_std": 7.988272654335229, + "n_dup_clusters_signed_per_threshold": { + "0.5": 129.66666666666666, + "0.6": 37.333333333333336, + "0.7": 11.333333333333334, + "0.8": 3.6666666666666665, + "0.9": 1.3333333333333333 + }, + "n_dup_clusters_signed_band": 13.416666666666668, + "n_dup_clusters_signed_band_std": 2.9825883613622133, + "largest_dup_signed_per_threshold": { + "0.5": 7.666666666666667, + "0.6": 7.666666666666667, + "0.7": 7.666666666666667, + "0.8": 7.666666666666667, + "0.9": 7.333333333333333 + }, + "largest_dup_signed_band": 7.583333333333333, + "largest_dup_signed_band_std": 2.742413778650722, + "dcs_signed_mean": 0.6871474385261536, + "dcs_signed_std": 0.0007623814310689849, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.677028311277455, + "0.6": 0.6843464302000939, + "0.7": 0.6863813224917976, + "0.8": 0.6867033839298529, + "0.9": 0.6868353836236768 + }, + "dedup_dcs_signed_band": 0.6860666300613553, + "dedup_dcs_signed_band_std": 0.0008536524428129008, + "matched_cos_signed_per_threshold": { + "0.5": 0.6849060455958048, + "0.6": 0.6869365175565084, + "0.7": 0.6874069571495056, + "0.8": 0.6872627337773641, + "0.9": 0.6872082750002543 + }, + "matched_cos_signed_band": 0.6872036208709081, + "matched_cos_signed_band_std": 0.0008725667190533835, + "unique_frac_signed_per_threshold": { + "0.5": 0.9884982638888888, + "0.6": 0.9962293836805557, + "0.7": 0.9985080295138888, + "0.8": 0.9991861979166666, + "0.9": 0.9994574652777778 + }, + "unique_frac_signed_band": 0.9983452690972222, + "unique_frac_signed_band_std": 0.00021491091413703044, + "matched_frac_signed_per_threshold": { + "0.5": 0.7305772569444443, + "0.6": 0.661865234375, + "0.7": 0.5732421875, + "0.8": 0.4565700954861111, + "0.9": 0.2611490885416667 + }, + "matched_frac_signed_band": 0.4882066514756945, + "matched_frac_signed_band_std": 0.0025838483139887848, + "babel_mu1_abs_mean": 0.33395642042160034, + "babel_mu1_abs_std": 0.001580666468696058, + "n_dup_abs_per_threshold": { + "0.5": 895.0, + "0.6": 642.3333333333334, + "0.7": 540.0, + "0.8": 464.3333333333333, + "0.9": 347.0 + }, + "n_dup_abs_band": 498.4166666666667, + "n_dup_abs_band_std": 9.928914005737655, + "n_dup_clusters_abs_per_threshold": { + "0.5": 438.0, + "0.6": 314.6666666666667, + "0.7": 265.6666666666667, + "0.8": 228.66666666666666, + "0.9": 170.66666666666666 + }, + "n_dup_clusters_abs_band": 244.91666666666666, + "n_dup_clusters_abs_band_std": 5.490522136676377, + "largest_dup_abs_per_threshold": { + "0.5": 7.666666666666667, + "0.6": 7.666666666666667, + "0.7": 7.666666666666667, + "0.8": 7.666666666666667, + "0.9": 7.333333333333333 + }, + "largest_dup_abs_band": 7.583333333333333, + "largest_dup_abs_band_std": 2.742413778650722, + "dcs_abs_mean": 0.6882694760958353, + "dcs_abs_std": 0.0007533812546554668, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6628897146583768, + "0.6": 0.6716415357465545, + "0.7": 0.6744589026865367, + "0.8": 0.6762837766064979, + "0.9": 0.6791833077861358 + }, + "dedup_dcs_abs_band": 0.6753918807064312, + "dedup_dcs_abs_band_std": 0.0006833803857438924, + "matched_cos_abs_per_threshold": { + "0.5": 0.6884954373041788, + "0.6": 0.6900418400764465, + "0.7": 0.6898602445920309, + "0.8": 0.6895075440406799, + "0.9": 0.689071516195933 + }, + "matched_cos_abs_band": 0.6896202862262726, + "matched_cos_abs_band_std": 0.0006406496395379985, + "unique_frac_abs_per_threshold": { + "0.5": 0.9628092447916666, + "0.6": 0.9733344184027777, + "0.7": 0.9776746961805555, + "0.8": 0.9808213975694445, + "0.9": 0.9856499565972222 + }, + "unique_frac_abs_band": 0.9793701171875, + "unique_frac_abs_band_std": 0.00021313809041095065, + "matched_frac_abs_per_threshold": { + "0.5": 0.7164984809027777, + "0.6": 0.652615017361111, + "0.7": 0.5656467013888888, + "0.8": 0.45046657986111116, + "0.9": 0.25800238715277773 + }, + "matched_frac_abs_band": 0.4816826714409722, + "matched_frac_abs_band_std": 0.00255574130740011, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 12288 + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/topk/x16/l0_32/code_metrics_step_final.json b/lm/gpt2-small_layer8/topk/x16/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..3fc2d2912cba1649ba45d498d3080fa9bd6a5c96 --- /dev/null +++ b/lm/gpt2-small_layer8/topk/x16/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 31.999998333333334, + "recon_r2": 0.9188660914184298, + "l1_per_token": 303.4745559390497 + }, + "seed1": { + "l0": 31.999998333333334, + "recon_r2": 0.9188968355009285, + "l1_per_token": 302.73255469888323 + }, + "seed2": { + "l0": 32.0, + "recon_r2": 0.9188311543690484, + "l1_per_token": 299.53091751129386 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7332379471760436, + "mcc_abs": 0.7332459770450033, + "act_iou": 0.42719283790782775, + "act_iou_n_pairs": 12288, + "input_r2": 0.9677071958795345, + "matched_mcc_signed": { + "0.6": 0.8334386103882669, + "0.7": 0.8624033895585348, + "0.8": 0.8921861177178851, + "0.9": 0.9253889427044499 + }, + "matched_mcc_abs": { + "0.6": 0.8319767127210952, + "0.7": 0.8620948277805379, + "0.8": 0.8918164865414169, + "0.9": 0.925234915399941 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.732708780735034, + "mcc_abs": 0.7327751645680166, + "act_iou": 0.4254377309713604, + "act_iou_n_pairs": 12288, + "input_r2": 0.9672168281260776, + "matched_mcc_signed": { + "0.6": 0.8296723283299835, + "0.7": 0.8599025981183658, + "0.8": 0.8900331003964368, + "0.9": 0.9241263359632728 + }, + "matched_mcc_abs": { + "0.6": 0.8286251284759365, + "0.7": 0.8595614815976133, + "0.8": 0.8898297556254628, + "0.9": 0.9240181050440064 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7337874363555313, + "mcc_abs": 0.7338014830545019, + "act_iou": 0.42634813651985365, + "act_iou_n_pairs": 12288, + "input_r2": 0.9670971169742509, + "matched_mcc_signed": { + "0.6": 0.8293677133703418, + "0.7": 0.8602499403116857, + "0.8": 0.8922489045522687, + "0.9": 0.9245633551030022 + }, + "matched_mcc_abs": { + "0.6": 0.8285712379823212, + "0.7": 0.8594488637616019, + "0.8": 0.8920901639754316, + "0.9": 0.9245641631443483 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 31.999998888888893, + "l0_std": 9.622504482710877e-07, + "recon_r2_mean": 0.9188646937628023, + "recon_r2_std": 3.286286433351162e-05, + "l1_per_token_mean": 301.9126760497423, + "l1_per_token_std": 2.095762811414034, + "input_r2_mean": 0.967340380326621, + "input_r2_std": 0.00032326139169769244, + "mcc_signed_mean": 0.733244721422203, + "mcc_signed_std": 0.0005393597173648647, + "mcc_abs_mean": 0.7332742082225073, + "mcc_abs_std": 0.000513741334197277, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8308262173628641, + "0.7": 0.8608519759961956, + "0.8": 0.8914893742221969, + "0.9": 0.924692877923575 + }, + "matched_mcc_signed_band": 0.8769651113762079, + "matched_mcc_abs_per_threshold": { + "0.6": 0.829724359726451, + "0.7": 0.8603683910465844, + "0.8": 0.8912454687141037, + "0.9": 0.9246057278627653 + }, + "matched_mcc_abs_band": 0.876485986837476, + "act_iou_mean": 0.42632623513301393, + "act_iou_std": 0.0008777584193091415, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/topk/x16/l0_32/dict_metrics_step_final.json b/lm/gpt2-small_layer8/topk/x16/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..af7a65d10bbded7f815ea6bbc79bfb0d314652b4 --- /dev/null +++ b/lm/gpt2-small_layer8/topk/x16/l0_32/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 31.999996875, + "recon_r2": 0.9187235385163666, + "l1_per_token": 303.2671258173008, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 17, + "dense_rate": 0.0013834635416666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1898, + "0.6": 582, + "0.7": 155, + "0.8": 40, + "0.9": 22 + }, + "n_dup_clusters_signed": { + "0.5": 886, + "0.6": 266, + "0.7": 65, + "0.8": 10, + "0.9": 3 + }, + "n_dup_removed_signed": { + "0.5": 1012, + "0.6": 316, + "0.7": 90, + "0.8": 30, + "0.9": 19 + }, + "largest_dup_signed": { + "0.5": 24, + "0.6": 24, + "0.7": 24, + "0.8": 22, + "0.9": 18 + }, + "babel_mu1_signed": 0.4127112329006195, + "n_dup_abs": { + "0.5": 2035, + "0.6": 691, + "0.7": 233, + "0.8": 103, + "0.9": 35 + }, + "n_dup_clusters_abs": { + "0.5": 949, + "0.6": 318, + "0.7": 103, + "0.8": 41, + "0.9": 9 + }, + "n_dup_removed_abs": { + "0.5": 1086, + "0.6": 373, + "0.7": 130, + "0.8": 62, + "0.9": 26 + }, + "largest_dup_abs": { + "0.5": 24, + "0.6": 24, + "0.7": 24, + "0.8": 22, + "0.9": 18 + }, + "babel_mu1_abs": 0.41923752427101135, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 31.999999365234373, + "recon_r2": 0.9187593952479948, + "l1_per_token": 302.4908204021793, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 15, + "dense_rate": 0.001220703125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1901, + "0.6": 608, + "0.7": 151, + "0.8": 30, + "0.9": 21 + }, + "n_dup_clusters_signed": { + "0.5": 900, + "0.6": 286, + "0.7": 64, + "0.8": 5, + "0.9": 1 + }, + "n_dup_removed_signed": { + "0.5": 1001, + "0.6": 322, + "0.7": 87, + "0.8": 25, + "0.9": 20 + }, + "largest_dup_signed": { + "0.5": 22, + "0.6": 22, + "0.7": 22, + "0.8": 22, + "0.9": 21 + }, + "babel_mu1_signed": 0.4121952950954437, + "n_dup_abs": { + "0.5": 2051, + "0.6": 709, + "0.7": 228, + "0.8": 93, + "0.9": 37 + }, + "n_dup_clusters_abs": { + "0.5": 972, + "0.6": 334, + "0.7": 102, + "0.8": 36, + "0.9": 9 + }, + "n_dup_removed_abs": { + "0.5": 1079, + "0.6": 375, + "0.7": 126, + "0.8": 57, + "0.9": 28 + }, + "largest_dup_abs": { + "0.5": 22, + "0.6": 22, + "0.7": 22, + "0.8": 22, + "0.9": 21 + }, + "babel_mu1_abs": 0.41910290718078613, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 31.99999990234375, + "recon_r2": 0.9186805598122308, + "l1_per_token": 299.460248829974, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 17, + "dense_rate": 0.0013834635416666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1888, + "0.6": 612, + "0.7": 168, + "0.8": 42, + "0.9": 25 + }, + "n_dup_clusters_signed": { + "0.5": 886, + "0.6": 284, + "0.7": 70, + "0.8": 10, + "0.9": 3 + }, + "n_dup_removed_signed": { + "0.5": 1002, + "0.6": 328, + "0.7": 98, + "0.8": 32, + "0.9": 22 + }, + "largest_dup_signed": { + "0.5": 24, + "0.6": 24, + "0.7": 24, + "0.8": 24, + "0.9": 17 + }, + "babel_mu1_signed": 0.41259634494781494, + "n_dup_abs": { + "0.5": 2028, + "0.6": 722, + "0.7": 237, + "0.8": 96, + "0.9": 41 + }, + "n_dup_clusters_abs": { + "0.5": 950, + "0.6": 336, + "0.7": 103, + "0.8": 36, + "0.9": 11 + }, + "n_dup_removed_abs": { + "0.5": 1078, + "0.6": 386, + "0.7": 134, + "0.8": 60, + "0.9": 30 + }, + "largest_dup_abs": { + "0.5": 24, + "0.6": 24, + "0.7": 24, + "0.8": 24, + "0.9": 17 + }, + "babel_mu1_abs": 0.41921886801719666, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8031067252159119, + "dcs_abs": 0.8039210438728333, + "frac_sign_flipped": 0.009114583333333334, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.792782723903656, + "0.6": 0.7996417880058289, + "0.7": 0.8023783564567566, + "0.8": 0.8030262589454651, + "0.9": 0.8028565645217896 + }, + "unique_frac_signed": { + "0.5": 0.9180908203125, + "0.6": 0.9740397135416666, + "0.7": 0.9927978515625, + "0.8": 0.9977620442708334, + "0.9": 0.9984130859375 + }, + "dedup_dcs_signed": { + "0.5": 0.7278465413182857, + "0.6": 0.7788828581251437, + "0.7": 0.7965995084305177, + "0.8": 0.8012291217285868, + "0.9": 0.8015825001493795 + }, + "matched_frac_signed": { + "0.5": 0.7880045572916666, + "0.6": 0.7965494791666666, + "0.7": 0.7562662760416666, + "0.8": 0.6643880208333334, + "0.9": 0.4951985677083333 + }, + "matched_cos_abs": { + "0.5": 0.7937320470809937, + "0.6": 0.8005757927894592, + "0.7": 0.8033550977706909, + "0.8": 0.8038648366928101, + "0.9": 0.8036137819290161 + }, + "unique_frac_abs": { + "0.5": 0.9119059244791666, + "0.6": 0.9695638020833334, + "0.7": 0.9895833333333334, + "0.8": 0.9951578776041666, + "0.9": 0.997802734375 + }, + "dedup_dcs_abs": { + "0.5": 0.7238089561821349, + "0.6": 0.776209309512827, + "0.7": 0.7949868155022463, + "0.8": 0.7999724247638369, + "0.9": 0.8018480289902072 + }, + "matched_frac_abs": { + "0.5": 0.783447265625, + "0.6": 0.7952473958333334, + "0.7": 0.7548014322916666, + "0.8": 0.663330078125, + "0.9": 0.4949544270833333 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.803264319896698, + "dcs_abs": 0.8043050765991211, + "frac_sign_flipped": 0.010498046875, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7928521037101746, + "0.6": 0.7999552488327026, + "0.7": 0.8027979731559753, + "0.8": 0.8030977845191956, + "0.9": 0.803120493888855 + }, + "unique_frac_signed": { + "0.5": 0.9180501302083334, + "0.6": 0.9737955729166666, + "0.7": 0.9923502604166666, + "0.8": 0.9974772135416666, + "0.9": 0.9983317057291666 + }, + "dedup_dcs_signed": { + "0.5": 0.7278779770470768, + "0.6": 0.7789928798447363, + "0.7": 0.7966567777233042, + "0.8": 0.801071740303693, + "0.9": 0.8017806525701113 + }, + "matched_frac_signed": { + "0.5": 0.7903645833333334, + "0.6": 0.7997233072916666, + "0.7": 0.7583821614583334, + "0.8": 0.665283203125, + "0.9": 0.4878743489583333 + }, + "matched_cos_abs": { + "0.5": 0.7938441634178162, + "0.6": 0.8010444045066833, + "0.7": 0.8035865426063538, + "0.8": 0.8041537404060364, + "0.9": 0.804135799407959 + }, + "unique_frac_abs": { + "0.5": 0.9119466145833334, + "0.6": 0.9691162109375, + "0.7": 0.9892578125, + "0.8": 0.9950358072916666, + "0.9": 0.9977213541666666 + }, + "dedup_dcs_abs": { + "0.5": 0.7239434973356159, + "0.6": 0.776305118088203, + "0.7": 0.7949542652931996, + "0.8": 0.8001617662715337, + "0.9": 0.8023034587192038 + }, + "matched_frac_abs": { + "0.5": 0.78564453125, + "0.6": 0.7965494791666666, + "0.7": 0.7565104166666666, + "0.8": 0.6646321614583334, + "0.9": 0.4874674479166667 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8033540844917297, + "dcs_abs": 0.8043012022972107, + "frac_sign_flipped": 0.009684244791666666, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7921372056007385, + "0.6": 0.7998843789100647, + "0.7": 0.8028441071510315, + "0.8": 0.803403377532959, + "0.9": 0.8032242655754089 + }, + "unique_frac_signed": { + "0.5": 0.9184977213541666, + "0.6": 0.9735514322916666, + "0.7": 0.9924723307291666, + "0.8": 0.9976806640625, + "0.9": 0.998291015625 + }, + "dedup_dcs_signed": { + "0.5": 0.7275762183441353, + "0.6": 0.7787285827556236, + "0.7": 0.796800562236361, + "0.8": 0.8015400152071379, + "0.9": 0.8018515678559197 + }, + "matched_frac_signed": { + "0.5": 0.7906087239583334, + "0.6": 0.8017578125, + "0.7": 0.7554524739583334, + "0.8": 0.658935546875, + "0.9": 0.493896484375 + }, + "matched_cos_abs": { + "0.5": 0.7933627367019653, + "0.6": 0.8010218143463135, + "0.7": 0.8037869334220886, + "0.8": 0.8043836951255798, + "0.9": 0.8041170835494995 + }, + "unique_frac_abs": { + "0.5": 0.9122314453125, + "0.6": 0.9690348307291666, + "0.7": 0.9894205729166666, + "0.8": 0.9952392578125, + "0.9": 0.9976399739583334 + }, + "dedup_dcs_abs": { + "0.5": 0.7237304359587142, + "0.6": 0.7762180382754498, + "0.7": 0.7952833281694135, + "0.8": 0.8005542317332583, + "0.9": 0.8022193462917737 + }, + "matched_frac_abs": { + "0.5": 0.78662109375, + "0.6": 0.7979329427083334, + "0.7": 0.7552083333333334, + "0.8": 0.658447265625, + "0.9": 0.4935709635416667 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 16, + "n_dense_std": 1.1547005383792515, + "l0_mean": 32, + "l0_std": 1.6152690116851594e-06, + "recon_r2_mean": 0.9187211645255308, + "recon_r2_std": 3.9471297893386325e-05, + "l1_per_token_mean": 301.73939834981803, + "l1_per_token_std": 2.011604937541344, + "babel_mu1_signed_mean": 0.4125009576479594, + "babel_mu1_signed_std": 0.00027087267753079136, + "n_dup_signed_per_threshold": { + "0.5": 1895.6666666666667, + "0.6": 600.6666666666666, + "0.7": 158.0, + "0.8": 37.333333333333336, + "0.9": 22.666666666666668 + }, + "n_dup_signed_band": 204.66666666666666, + "n_dup_signed_band_std": 6.286559737514098, + "n_dup_clusters_signed_per_threshold": { + "0.5": 890.6666666666666, + "0.6": 278.6666666666667, + "0.7": 66.33333333333333, + "0.8": 8.333333333333334, + "0.9": 2.3333333333333335 + }, + "n_dup_clusters_signed_band": 88.91666666666666, + "n_dup_clusters_signed_band_std": 2.8759056544562327, + "largest_dup_signed_per_threshold": { + "0.5": 23.333333333333332, + "0.6": 23.333333333333332, + "0.7": 23.333333333333332, + "0.8": 22.666666666666668, + "0.9": 18.666666666666668 + }, + "largest_dup_signed_band": 22.0, + "largest_dup_signed_band_std": 0.25, + "dcs_signed_mean": 0.8032417098681132, + "dcs_signed_std": 0.00012522005780920024, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7277669122364993, + "0.6": 0.7788681069085012, + "0.7": 0.796685616130061, + "0.8": 0.8012802924131393, + "0.9": 0.8017382401918035 + }, + "dedup_dcs_signed_band": 0.7946430639108762, + "dedup_dcs_signed_band_std": 7.980335833227631e-05, + "matched_cos_signed_per_threshold": { + "0.5": 0.7925906777381897, + "0.6": 0.7998271385828654, + "0.7": 0.8026734789212545, + "0.8": 0.8031758069992065, + "0.9": 0.8030671079953512 + }, + "matched_cos_signed_band": 0.8021858831246694, + "matched_cos_signed_band_std": 0.00018823131032600743, + "unique_frac_signed_per_threshold": { + "0.5": 0.918212890625, + "0.6": 0.9737955729166666, + "0.7": 0.9925401475694443, + "0.8": 0.9976399739583334, + "0.9": 0.9983452690972222 + }, + "unique_frac_signed_band": 0.9905802408854166, + "unique_frac_signed_band_std": 0.00014985066592060673, + "matched_frac_signed_per_threshold": { + "0.5": 0.7896592881944445, + "0.6": 0.799343532986111, + "0.7": 0.7567003038194445, + "0.8": 0.6628689236111112, + "0.9": 0.4923231336805555 + }, + "matched_frac_signed_band": 0.6778089735243056, + "matched_frac_signed_band_std": 0.0002950617122098218, + "babel_mu1_abs_mean": 0.41918643315633136, + "babel_mu1_abs_std": 7.293459669243775e-05, + "n_dup_abs_per_threshold": { + "0.5": 2038.0, + "0.6": 707.3333333333334, + "0.7": 232.66666666666666, + "0.8": 97.33333333333333, + "0.9": 37.666666666666664 + }, + "n_dup_abs_band": 268.75, + "n_dup_abs_band_std": 4.589389937671455, + "n_dup_clusters_abs_per_threshold": { + "0.5": 957.0, + "0.6": 329.3333333333333, + "0.7": 102.66666666666667, + "0.8": 37.666666666666664, + "0.9": 9.666666666666666 + }, + "n_dup_clusters_abs_band": 119.83333333333334, + "n_dup_clusters_abs_band_std": 1.9094065395649333, + "largest_dup_abs_per_threshold": { + "0.5": 23.333333333333332, + "0.6": 23.333333333333332, + "0.7": 23.333333333333332, + "0.8": 22.666666666666668, + "0.9": 18.666666666666668 + }, + "largest_dup_abs_band": 22.0, + "largest_dup_abs_band_std": 0.25, + "dcs_abs_mean": 0.8041757742563883, + "dcs_abs_std": 0.00022061148833054314, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.7238276298254883, + "0.6": 0.7762441552921601, + "0.7": 0.7950748029882865, + "0.8": 0.8002294742562096, + "0.9": 0.8021236113337281 + }, + "dedup_dcs_abs_band": 0.793418010967596, + "dedup_dcs_abs_band_std": 0.0001577068739301806, + "matched_cos_abs_per_threshold": { + "0.5": 0.7936463157335917, + "0.6": 0.8008806705474854, + "0.7": 0.8035761912663778, + "0.8": 0.8041340907414755, + "0.9": 0.8039555549621582 + }, + "matched_cos_abs_band": 0.8031366268793743, + "matched_cos_abs_band_std": 0.00025092477054536165, + "unique_frac_abs_per_threshold": { + "0.5": 0.9120279947916666, + "0.6": 0.96923828125, + "0.7": 0.9894205729166666, + "0.8": 0.995144314236111, + "0.9": 0.9977213541666666 + }, + "unique_frac_abs_band": 0.987881130642361, + "unique_frac_abs_band_std": 0.00012880737245843396, + "matched_frac_abs_per_threshold": { + "0.5": 0.7852376302083334, + "0.6": 0.7965766059027778, + "0.7": 0.7555067274305555, + "0.8": 0.6621365017361112, + "0.9": 0.49199761284722227 + }, + "matched_frac_abs_band": 0.6765543619791667, + "matched_frac_abs_band_std": 0.00045810263058262084, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 12288 + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/topk/x16/l0_64/code_metrics_step_final.json b/lm/gpt2-small_layer8/topk/x16/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..4f292d556a82c5ff316b860bb1604558daee7c14 --- /dev/null +++ b/lm/gpt2-small_layer8/topk/x16/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 64.0, + "recon_r2": 0.9367044690599102, + "l1_per_token": 402.47130423717385 + }, + "seed1": { + "l0": 64.0, + "recon_r2": 0.9367372707274907, + "l1_per_token": 405.50469941072276 + }, + "seed2": { + "l0": 64.0, + "recon_r2": 0.936667890720555, + "l1_per_token": 404.5670342416222 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7154791675287763, + "mcc_abs": 0.7155000917793884, + "act_iou": 0.38432138281572364, + "act_iou_n_pairs": 12288, + "input_r2": 0.9702922032566944, + "matched_mcc_signed": { + "0.6": 0.8396774415419663, + "0.7": 0.8702603970305968, + "0.8": 0.9003272158089278, + "0.9": 0.931590872551593 + }, + "matched_mcc_abs": { + "0.6": 0.8394830328620485, + "0.7": 0.8701937860016804, + "0.8": 0.8999517704283748, + "0.9": 0.9314596600430127 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7152980081654632, + "mcc_abs": 0.7153117201294469, + "act_iou": 0.3829919118094738, + "act_iou_n_pairs": 12288, + "input_r2": 0.9701611407354621, + "matched_mcc_signed": { + "0.6": 0.8371483846963703, + "0.7": 0.8690563870765948, + "0.8": 0.8998990071087274, + "0.9": 0.9320397553804204 + }, + "matched_mcc_abs": { + "0.6": 0.8360956723617564, + "0.7": 0.8686847225019604, + "0.8": 0.8996041919304434, + "0.9": 0.9318352574202757 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7160773360235737, + "mcc_abs": 0.7161079586126432, + "act_iou": 0.3854065847352393, + "act_iou_n_pairs": 12288, + "input_r2": 0.9703792896036836, + "matched_mcc_signed": { + "0.6": 0.8396184172971703, + "0.7": 0.8704120671109851, + "0.8": 0.8999309930951622, + "0.9": 0.932475407799723 + }, + "matched_mcc_abs": { + "0.6": 0.8393703207918278, + "0.7": 0.8699815324689788, + "0.8": 0.8998374421548812, + "0.9": 0.9323171002691466 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 64.0, + "l0_std": 0.0, + "recon_r2_mean": 0.9367032101693186, + "recon_r2_std": 3.4707131035854996e-05, + "l1_per_token_mean": 404.18101262983964, + "l1_per_token_std": 1.5531036936727305, + "input_r2_mean": 0.9702775445319466, + "input_r2_std": 0.00010981070455560634, + "mcc_signed_mean": 0.7156181705726045, + "mcc_signed_std": 0.00040783496938332154, + "mcc_abs_mean": 0.7156399235071595, + "mcc_abs_std": 0.00041612932436836875, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8388147478451691, + "0.7": 0.8699096170727256, + "0.8": 0.9000524053376058, + "0.9": 0.9320353452439122 + }, + "matched_mcc_signed_band": 0.8852030288748531, + "matched_mcc_abs_per_threshold": { + "0.6": 0.838316342005211, + "0.7": 0.8696200136575398, + "0.8": 0.8997978015045666, + "0.9": 0.9318706725774785 + }, + "matched_mcc_abs_band": 0.884901207436199, + "act_iou_mean": 0.38423995978681225, + "act_iou_std": 0.00120939390474489, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/topk/x16/l0_64/dict_metrics_step_final.json b/lm/gpt2-small_layer8/topk/x16/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..8305fd2396332c42c5c688b79110b4305f58234d --- /dev/null +++ b/lm/gpt2-small_layer8/topk/x16/l0_64/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 64.0, + "recon_r2": 0.9365965701152947, + "l1_per_token": 402.35637088713605, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 50, + "dense_rate": 0.004069010416666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 753, + "0.6": 261, + "0.7": 107, + "0.8": 48, + "0.9": 38 + }, + "n_dup_clusters_signed": { + "0.5": 346, + "0.6": 109, + "0.7": 35, + "0.8": 6, + "0.9": 2 + }, + "largest_dup_signed": { + "0.5": 38, + "0.6": 38, + "0.7": 38, + "0.8": 38, + "0.9": 36 + }, + "babel_mu1_signed": 0.3429441452026367, + "n_dup_abs": { + "0.5": 1103, + "0.6": 556, + "0.7": 364, + "0.8": 252, + "0.9": 156 + }, + "n_dup_clusters_abs": { + "0.5": 513, + "0.6": 252, + "0.7": 161, + "0.8": 108, + "0.9": 61 + }, + "largest_dup_abs": { + "0.5": 38, + "0.6": 38, + "0.7": 38, + "0.8": 38, + "0.9": 36 + }, + "babel_mu1_abs": 0.36265507340431213 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 63.99999995117187, + "recon_r2": 0.9366311205453229, + "l1_per_token": 405.3546329785476, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 53, + "dense_rate": 0.004313151041666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 709, + "0.6": 220, + "0.7": 90, + "0.8": 49, + "0.9": 36 + }, + "n_dup_clusters_signed": { + "0.5": 325, + "0.6": 90, + "0.7": 27, + "0.8": 7, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 37, + "0.6": 37, + "0.7": 37, + "0.8": 37, + "0.9": 36 + }, + "babel_mu1_signed": 0.3416374623775482, + "n_dup_abs": { + "0.5": 1053, + "0.6": 522, + "0.7": 365, + "0.8": 264, + "0.9": 168 + }, + "n_dup_clusters_abs": { + "0.5": 494, + "0.6": 239, + "0.7": 163, + "0.8": 114, + "0.9": 67 + }, + "largest_dup_abs": { + "0.5": 37, + "0.6": 37, + "0.7": 37, + "0.8": 37, + "0.9": 36 + }, + "babel_mu1_abs": 0.36193737387657166 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 64.0, + "recon_r2": 0.9365662586589976, + "l1_per_token": 404.4139333175605, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 53, + "dense_rate": 0.004313151041666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 760, + "0.6": 220, + "0.7": 84, + "0.8": 48, + "0.9": 33 + }, + "n_dup_clusters_signed": { + "0.5": 349, + "0.6": 91, + "0.7": 24, + "0.8": 8, + "0.9": 2 + }, + "largest_dup_signed": { + "0.5": 34, + "0.6": 34, + "0.7": 34, + "0.8": 34, + "0.9": 24 + }, + "babel_mu1_signed": 0.34213176369667053, + "n_dup_abs": { + "0.5": 1136, + "0.6": 551, + "0.7": 368, + "0.8": 269, + "0.9": 161 + }, + "n_dup_clusters_abs": { + "0.5": 533, + "0.6": 254, + "0.7": 165, + "0.8": 118, + "0.9": 66 + }, + "largest_dup_abs": { + "0.5": 34, + "0.6": 34, + "0.7": 34, + "0.8": 34, + "0.9": 24 + }, + "babel_mu1_abs": 0.36279430985450745 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.7563708424568176, + "dcs_abs": 0.7573065757751465, + "frac_sign_flipped": 0.013102213541666666, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7510197758674622, + "0.6": 0.7546582818031311, + "0.7": 0.755803108215332, + "0.8": 0.7556670904159546, + "0.9": 0.7557271718978882 + }, + "unique_frac_signed": { + "0.5": 0.9678141276041666, + "0.6": 0.988525390625, + "0.7": 0.9945068359375, + "0.8": 0.99658203125, + "0.9": 0.9971110026041666 + }, + "dedup_dcs_signed": { + "0.5": 0.7268475491946447, + "0.6": 0.7459988728078315, + "0.7": 0.7516513577429578, + "0.8": 0.7530842439155094, + "0.9": 0.7535438780663146 + }, + "matched_frac_signed": { + "0.5": 0.7954915364583334, + "0.6": 0.7548828125, + "0.7": 0.678955078125, + "0.8": 0.566650390625, + "0.9": 0.3814290364583333 + }, + "matched_cos_abs": { + "0.5": 0.7538285851478577, + "0.6": 0.7569456100463867, + "0.7": 0.7575489282608032, + "0.8": 0.7573351860046387, + "0.9": 0.7570705413818359 + }, + "unique_frac_abs": { + "0.5": 0.9532470703125, + "0.6": 0.9761149088541666, + "0.7": 0.9835205078125, + "0.8": 0.988037109375, + "0.9": 0.9920247395833334 + }, + "dedup_dcs_abs": { + "0.5": 0.7185848903100123, + "0.6": 0.7388658951579903, + "0.7": 0.7450649066158803, + "0.8": 0.7482752680080011, + "0.9": 0.751032706660529 + }, + "matched_frac_abs": { + "0.5": 0.7871907552083334, + "0.6": 0.7490234375, + "0.7": 0.6746419270833334, + "0.8": 0.5642903645833334, + "0.9": 0.3798014322916667 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.7565650343894958, + "dcs_abs": 0.7574143409729004, + "frac_sign_flipped": 0.013671875, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7503871321678162, + "0.6": 0.7551637887954712, + "0.7": 0.756263017654419, + "0.8": 0.7560281157493591, + "0.9": 0.7562129497528076 + }, + "unique_frac_signed": { + "0.5": 0.9667154947916666, + "0.6": 0.9885660807291666, + "0.7": 0.99462890625, + "0.8": 0.9966634114583334, + "0.9": 0.9972737630208334 + }, + "dedup_dcs_signed": { + "0.5": 0.7254108677589102, + "0.6": 0.7465293069981271, + "0.7": 0.7522010580869392, + "0.8": 0.753505561001172, + "0.9": 0.7541513340450668 + }, + "matched_frac_signed": { + "0.5": 0.798095703125, + "0.6": 0.7569986979166666, + "0.7": 0.6778157552083334, + "0.8": 0.5636393229166666, + "0.9": 0.38037109375 + }, + "matched_cos_abs": { + "0.5": 0.7533511519432068, + "0.6": 0.7570448517799377, + "0.7": 0.7578163743019104, + "0.8": 0.7577614188194275, + "0.9": 0.7572369575500488 + }, + "unique_frac_abs": { + "0.5": 0.9514567057291666, + "0.6": 0.9755452473958334, + "0.7": 0.9834798177083334, + "0.8": 0.9879964192708334, + "0.9": 0.9922688802083334 + }, + "dedup_dcs_abs": { + "0.5": 0.7167810052851564, + "0.6": 0.7385315072194014, + "0.7": 0.745297109654833, + "0.8": 0.7486655684551806, + "0.9": 0.7513826679205522 + }, + "matched_frac_abs": { + "0.5": 0.7874348958333334, + "0.6": 0.751220703125, + "0.7": 0.673828125, + "0.8": 0.5608723958333334, + "0.9": 0.3793131510416667 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.7570788264274597, + "dcs_abs": 0.7578358054161072, + "frac_sign_flipped": 0.013427734375, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.752122163772583, + "0.6": 0.7553932070732117, + "0.7": 0.7565862536430359, + "0.8": 0.7566334009170532, + "0.9": 0.7567099332809448 + }, + "unique_frac_signed": { + "0.5": 0.9676513671875, + "0.6": 0.9894612630208334, + "0.7": 0.9949951171875, + "0.8": 0.9966634114583334, + "0.9": 0.997314453125 + }, + "dedup_dcs_signed": { + "0.5": 0.7277920400665607, + "0.6": 0.7474323167480179, + "0.7": 0.7527996281060041, + "0.8": 0.7541088265813112, + "0.9": 0.7546777532843407 + }, + "matched_frac_signed": { + "0.5": 0.7975260416666666, + "0.6": 0.757080078125, + "0.7": 0.6797688802083334, + "0.8": 0.5694173177083334, + "0.9": 0.3841145833333333 + }, + "matched_cos_abs": { + "0.5": 0.7552061676979065, + "0.6": 0.7581255435943604, + "0.7": 0.7584744095802307, + "0.8": 0.7581374049186707, + "0.9": 0.7576844692230225 + }, + "unique_frac_abs": { + "0.5": 0.9527180989583334, + "0.6": 0.9763997395833334, + "0.7": 0.9835205078125, + "0.8": 0.9877522786458334, + "0.9": 0.9920247395833334 + }, + "dedup_dcs_abs": { + "0.5": 0.7194985844107578, + "0.6": 0.7402335833370065, + "0.7": 0.7459751364731346, + "0.8": 0.7488519492350558, + "0.9": 0.7516417382673051 + }, + "matched_frac_abs": { + "0.5": 0.7875162760416666, + "0.6": 0.7505696614583334, + "0.7": 0.6756184895833334, + "0.8": 0.5664876302083334, + "0.9": 0.3829752604166667 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 52, + "n_dense_std": 1.7320508075688772, + "l0_mean": 64, + "l0_std": 2.819093245177222e-08, + "recon_r2_mean": 0.9365979831065383, + "recon_r2_std": null, + "l1_per_token_mean": 404.04164572774806, + "l1_per_token_std": 1.5334087613370364, + "babel_mu1_signed_mean": 0.34223779042561847, + "babel_mu1_signed_std": 0.000659762269142355, + "n_dup_signed_per_threshold": { + "0.5": 740.6666666666666, + "0.6": 233.66666666666666, + "0.7": 93.66666666666667, + "0.8": 48.333333333333336, + "0.9": 35.666666666666664 + }, + "n_dup_signed_band": 102.83333333333333, + "n_dup_signed_band_std": 9.32179346120334, + "n_dup_clusters_signed_per_threshold": { + "0.5": 340.0, + "0.6": 96.66666666666667, + "0.7": 28.666666666666668, + "0.8": 7.0, + "0.9": 1.6666666666666667 + }, + "n_dup_clusters_signed_band": 33.5, + "n_dup_clusters_signed_band_std": 3.897114317029974, + "largest_dup_signed_per_threshold": { + "0.5": 36.333333333333336, + "0.6": 36.333333333333336, + "0.7": 36.333333333333336, + "0.8": 36.333333333333336, + "0.9": 32.0 + }, + "largest_dup_signed_band": 35.25, + "largest_dup_signed_band_std": 3.2691742076555053, + "dcs_signed_mean": 0.7566715677579244, + "dcs_signed_std": 0.00036581736510598353, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7266834856733718, + "0.6": 0.7466534988513255, + "0.7": 0.7522173479786337, + "0.8": 0.7535662104993309, + "0.9": 0.754124321798574 + }, + "dedup_dcs_signed_band": 0.7516403447819661, + "dedup_dcs_signed_band_std": 0.0005937195339272511, + "matched_cos_signed_per_threshold": { + "0.5": 0.7511763572692871, + "0.6": 0.755071759223938, + "0.7": 0.7562174598375956, + "0.8": 0.7561095356941223, + "0.9": 0.7562166849772135 + }, + "matched_cos_signed_band": 0.7559038599332173, + "matched_cos_signed_band_std": 0.00043354146817818623, + "unique_frac_signed_per_threshold": { + "0.5": 0.9673936631944443, + "0.6": 0.9888509114583334, + "0.7": 0.9947102864583334, + "0.8": 0.9966362847222223, + "0.9": 0.9972330729166666 + }, + "unique_frac_signed_band": 0.994357638888889, + "unique_frac_signed_band_std": 0.00022317820464284971, + "matched_frac_signed_per_threshold": { + "0.5": 0.7970377604166666, + "0.6": 0.7563205295138888, + "0.7": 0.6788465711805557, + "0.8": 0.5665690104166666, + "0.9": 0.3819715711805555 + }, + "matched_frac_signed_band": 0.5959269205729166, + "matched_frac_signed_band_std": 0.0014956035115144845, + "babel_mu1_abs_mean": 0.36246225237846375, + "babel_mu1_abs_std": 0.00045985843319125, + "n_dup_abs_per_threshold": { + "0.5": 1097.3333333333333, + "0.6": 543.0, + "0.7": 365.6666666666667, + "0.8": 261.6666666666667, + "0.9": 161.66666666666666 + }, + "n_dup_abs_band": 333.00000000000006, + "n_dup_abs_band_std": 3.848701079585163, + "n_dup_clusters_abs_per_threshold": { + "0.5": 513.3333333333334, + "0.6": 248.33333333333334, + "0.7": 163.0, + "0.8": 113.33333333333333, + "0.9": 64.66666666666667 + }, + "n_dup_clusters_abs_band": 147.33333333333334, + "n_dup_clusters_abs_band_std": 2.9615592739861434, + "largest_dup_abs_per_threshold": { + "0.5": 36.333333333333336, + "0.6": 36.333333333333336, + "0.7": 36.333333333333336, + "0.8": 36.333333333333336, + "0.9": 32.0 + }, + "largest_dup_abs_band": 35.25, + "largest_dup_abs_band_std": 3.2691742076555053, + "dcs_abs_mean": 0.7575189073880514, + "dcs_abs_std": 0.0002796812553622218, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.7182881600019755, + "0.6": 0.7392103285714661, + "0.7": 0.7454457175812826, + "0.8": 0.7485975952327459, + "0.9": 0.7513523709494621 + }, + "dedup_dcs_abs_band": 0.7461515030837391, + "dedup_dcs_abs_band_std": 0.0004608375140367449, + "matched_cos_abs_per_threshold": { + "0.5": 0.754128634929657, + "0.6": 0.7573720018068949, + "0.7": 0.7579465707143148, + "0.8": 0.7577446699142456, + "0.9": 0.7573306560516357 + }, + "matched_cos_abs_band": 0.7575984746217728, + "matched_cos_abs_band_std": 0.00045514099847284806, + "unique_frac_abs_per_threshold": { + "0.5": 0.9524739583333334, + "0.6": 0.9760199652777778, + "0.7": 0.9835069444444445, + "0.8": 0.9879286024305557, + "0.9": 0.9921061197916666 + }, + "unique_frac_abs_band": 0.9848904079861112, + "unique_frac_abs_band_std": 5.873110648495796e-05, + "matched_frac_abs_per_threshold": { + "0.5": 0.7873806423611112, + "0.6": 0.7502712673611112, + "0.7": 0.6746961805555557, + "0.8": 0.5638834635416666, + "0.9": 0.3806966145833333 + }, + "matched_frac_abs_band": 0.5923868815104167, + "matched_frac_abs_band_std": 0.0013585559916012547, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/topk/x16/l0_64/seed0/config.json b/lm/gpt2-small_layer8/topk/x16/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2ec283bbdfc3763fb53f6abcf904f567d99d78d3 --- /dev/null +++ b/lm/gpt2-small_layer8/topk/x16/l0_64/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "topk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 64, + "top_k_inference": 0, + "class_type": "TopKSAE", + "module_name": "consdl.models.topk" +} diff --git a/lm/gpt2-small_layer8/topk/x16/l0_64/seed1/config.json b/lm/gpt2-small_layer8/topk/x16/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b99c6b5c2f7a7ef88a26acb8033a52c0c67b6093 --- /dev/null +++ b/lm/gpt2-small_layer8/topk/x16/l0_64/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "topk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 64, + "top_k_inference": 0, + "class_type": "TopKSAE", + "module_name": "consdl.models.topk" +} diff --git a/lm/gpt2-small_layer8/topk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/gpt2-small_layer8/topk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b36f2ef3d97931793a9d7d4330b7732ed8b64811 --- /dev/null +++ b/lm/gpt2-small_layer8/topk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.99999995117187, + "l0_per_token_0.001": 63.99999995117187, + "l0_per_token_0.01": 63.99999995117187, + "l0_per_token_0.05": 63.99999770507812, + "l0_per_token_0.1": 63.99993725585937, + "l0_per_token_0.5": 63.97626918945313, + "l0_per_token_1.0": 63.95895239257813, + "l1_per_token": 405.3546329785476, + "r2_score": 0.9366311205453229, + "l2_loss": 1.486154452634491 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/topk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/gpt2-small_layer8/topk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b2dfaf318f69f286c2b65f55156edf498d86c040 --- /dev/null +++ b/lm/gpt2-small_layer8/topk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "openai-community/gpt2", + "hook_target_name": "transformer.h.8", + "tokenizer_hf_name": "openai-community/gpt2", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/gpt2-small_layer8/topk/x16/l0_64/seed2/config.json b/lm/gpt2-small_layer8/topk/x16/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..679cca0712fbd39b44ba03ff81168d8177cb2858 --- /dev/null +++ b/lm/gpt2-small_layer8/topk/x16/l0_64/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "topk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 64, + "top_k_inference": 0, + "class_type": "TopKSAE", + "module_name": "consdl.models.topk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/code_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..7a968636e88a9128f1529100a03409aae005a389 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 128.90460833333333, + "recon_r2": 0.9541631364250042, + "l1_per_token": 110.57075750161553 + }, + "seed1": { + "l0": 129.38584666666668, + "recon_r2": 0.953496801809455, + "l1_per_token": 110.95171340782166 + }, + "seed2": { + "l0": 128.30222666666666, + "recon_r2": 0.9540735426620095, + "l1_per_token": 110.65353895970851 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.5970078119242745, + "mcc_abs": 0.5970201635812786, + "act_iou": 0.28834764846184324, + "act_iou_n_pairs": 12288, + "input_r2": 0.9692261393169346, + "matched_mcc_signed": { + "0.6": 0.8367494986463442, + "0.7": 0.8721502869809594, + "0.8": 0.9036672686877324, + "0.9": 0.9382183323647211 + }, + "matched_mcc_abs": { + "0.6": 0.837698836507457, + "0.7": 0.8732922534808946, + "0.8": 0.904321930352434, + "0.9": 0.9390959848413675 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5972517898761561, + "mcc_abs": 0.5972609121639602, + "act_iou": 0.2900967086639489, + "act_iou_n_pairs": 12288, + "input_r2": 0.9702486031885597, + "matched_mcc_signed": { + "0.6": 0.8368918059349358, + "0.7": 0.8731547078498, + "0.8": 0.9050075975691643, + "0.9": 0.9386224867723717 + }, + "matched_mcc_abs": { + "0.6": 0.8367509054319362, + "0.7": 0.8737330930665512, + "0.8": 0.905925200156982, + "0.9": 0.9398084167158419 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.595006520799397, + "mcc_abs": 0.5950190848917716, + "act_iou": 0.28768573600772895, + "act_iou_n_pairs": 12288, + "input_r2": 0.9693223912348543, + "matched_mcc_signed": { + "0.6": 0.8367881972582599, + "0.7": 0.8716665372141315, + "0.8": 0.9035238450222086, + "0.9": 0.9390517531487761 + }, + "matched_mcc_abs": { + "0.6": 0.8370477787824585, + "0.7": 0.8719841241560867, + "0.8": 0.9046613987900819, + "0.9": 0.9399885929072557 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 128.8642272222222, + "l0_std": 0.5429374289003651, + "recon_r2_mean": 0.9539111602988228, + "recon_r2_std": 0.0003616303069093279, + "l1_per_token_mean": 110.72533662304856, + "l1_per_token_std": 0.20036973065847896, + "input_r2_mean": 0.9695990445801161, + "input_r2_std": 0.0005645891402410028, + "mcc_signed_mean": 0.5964220408666092, + "mcc_signed_std": 0.0012319310474866235, + "mcc_abs_mean": 0.5964333868790034, + "mcc_abs_std": 0.0012307223705335898, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8368098339465133, + "0.7": 0.8723238440149638, + "0.8": 0.9040662370930351, + "0.9": 0.9386308574286231 + }, + "matched_mcc_signed_band": 0.8879576931207839, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8371658402406172, + "0.7": 0.8730031569011775, + "0.8": 0.9049695097664993, + "0.9": 0.9396309981548217 + }, + "matched_mcc_abs_band": 0.8886923762657789, + "act_iou_mean": 0.28871003104450704, + "act_iou_std": 0.001245667748407894, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/dict_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..869456c1d9d524b648de692c8f536675b22e58dc --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 129, + "l0_raw": 128.62628237304688, + "recon_r2": 0.9543583405070089, + "l1_per_token": 110.25429557056879, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 302, + "dense_rate": 0.024576822916666668, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 232, + "0.6": 44, + "0.7": 9, + "0.8": 4, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 112, + "0.6": 20, + "0.7": 4, + "0.8": 2, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 120, + "0.6": 24, + "0.7": 5, + "0.8": 2, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 5, + "0.7": 3, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.26621484756469727, + "n_dup_abs": { + "0.5": 992, + "0.6": 718, + "0.7": 599, + "0.8": 516, + "0.9": 418 + }, + "n_dup_clusters_abs": { + "0.5": 492, + "0.6": 357, + "0.7": 299, + "0.8": 258, + "0.9": 209 + }, + "n_dup_removed_abs": { + "0.5": 500, + "0.6": 361, + "0.7": 300, + "0.8": 258, + "0.9": 209 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 5, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.32800179719924927, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 129, + "l0_raw": 129.00770043945312, + "recon_r2": 0.9537419371398657, + "l1_per_token": 110.57495848696888, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 313, + "dense_rate": 0.025472005208333332, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 261, + "0.6": 68, + "0.7": 14, + "0.8": 4, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 126, + "0.6": 31, + "0.7": 6, + "0.8": 2, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 135, + "0.6": 37, + "0.7": 8, + "0.8": 2, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 7, + "0.6": 6, + "0.7": 4, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.26864972710609436, + "n_dup_abs": { + "0.5": 1024, + "0.6": 736, + "0.7": 612, + "0.8": 520, + "0.9": 406 + }, + "n_dup_clusters_abs": { + "0.5": 506, + "0.6": 365, + "0.7": 305, + "0.8": 260, + "0.9": 203 + }, + "n_dup_removed_abs": { + "0.5": 518, + "0.6": 371, + "0.7": 307, + "0.8": 260, + "0.9": 203 + }, + "largest_dup_abs": { + "0.5": 7, + "0.6": 6, + "0.7": 4, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3302949368953705, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 128, + "l0_raw": 127.99916665039062, + "recon_r2": 0.969080690975299, + "l1_per_token": 110.34473302497086, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 311, + "dense_rate": 0.025309244791666668, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 231, + "0.6": 68, + "0.7": 16, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 108, + "0.6": 31, + "0.7": 6, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 123, + "0.6": 37, + "0.7": 10, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 8, + "0.6": 8, + "0.7": 6, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.26729610562324524, + "n_dup_abs": { + "0.5": 964, + "0.6": 694, + "0.7": 580, + "0.8": 520, + "0.9": 430 + }, + "n_dup_clusters_abs": { + "0.5": 474, + "0.6": 344, + "0.7": 288, + "0.8": 260, + "0.9": 215 + }, + "n_dup_removed_abs": { + "0.5": 490, + "0.6": 350, + "0.7": 292, + "0.8": 260, + "0.9": 215 + }, + "largest_dup_abs": { + "0.5": 8, + "0.6": 8, + "0.7": 6, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.32813647389411926, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.6342378854751587, + "dcs_abs": 0.635639488697052, + "frac_sign_flipped": 0.034749348958333336, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.6310437321662903, + "0.6": 0.6337791085243225, + "0.7": 0.634257435798645, + "0.8": 0.6341849565505981, + "0.9": 0.6342378258705139 + }, + "unique_frac_signed": { + "0.5": 0.9896240234375, + "0.6": 0.9975179036458334, + "0.7": 0.9994710286458334, + "0.8": 0.9998372395833334, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6244960371914203, + "0.6": 0.6322060077097074, + "0.7": 0.6339219317839404, + "0.8": 0.6340817363428263, + "0.9": 0.6342378258705139 + }, + "matched_frac_signed": { + "0.5": 0.6543782552083334, + "0.6": 0.5874837239583334, + "0.7": 0.4986979166666667, + "0.8": 0.39501953125, + "0.9": 0.22298177083333334 + }, + "matched_cos_abs": { + "0.5": 0.6324643492698669, + "0.6": 0.6340935826301575, + "0.7": 0.6345424652099609, + "0.8": 0.6342830061912537, + "0.9": 0.6346820592880249 + }, + "unique_frac_abs": { + "0.5": 0.9585774739583334, + "0.6": 0.97021484375, + "0.7": 0.9753011067708334, + "0.8": 0.9789225260416666, + "0.9": 0.9832356770833334 + }, + "dedup_dcs_abs": { + "0.5": 0.6062660782918101, + "0.6": 0.6152070061943959, + "0.7": 0.618869968612368, + "0.8": 0.6209139226460441, + "0.9": 0.6240420442967055 + }, + "matched_frac_abs": { + "0.5": 0.6353352864583334, + "0.6": 0.5731608072916666, + "0.7": 0.4874674479166667, + "0.8": 0.38671875, + "0.9": 0.21769205729166666 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.6335923671722412, + "dcs_abs": 0.6349629759788513, + "frac_sign_flipped": 0.034912109375, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.6304008364677429, + "0.6": 0.6331819891929626, + "0.7": 0.6336492300033569, + "0.8": 0.6335591077804565, + "0.9": 0.633592426776886 + }, + "unique_frac_signed": { + "0.5": 0.9901123046875, + "0.6": 0.9975179036458334, + "0.7": 0.9993896484375, + "0.8": 0.9998779296875, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6241676250720047, + "0.6": 0.6316103704860628, + "0.7": 0.6332624812057475, + "0.8": 0.6334817690221826, + "0.9": 0.633592426776886 + }, + "matched_frac_signed": { + "0.5": 0.6522623697916666, + "0.6": 0.58984375, + "0.7": 0.502685546875, + "0.8": 0.392578125, + "0.9": 0.22314453125 + }, + "matched_cos_abs": { + "0.5": 0.6320589184761047, + "0.6": 0.6335814595222473, + "0.7": 0.6338989734649658, + "0.8": 0.6334912776947021, + "0.9": 0.6337850093841553 + }, + "unique_frac_abs": { + "0.5": 0.959716796875, + "0.6": 0.9710693359375, + "0.7": 0.9759114583333334, + "0.8": 0.9789225260416666, + "0.9": 0.9827473958333334 + }, + "dedup_dcs_abs": { + "0.5": 0.606597560676164, + "0.6": 0.6152515271605807, + "0.7": 0.6186292716301978, + "0.8": 0.6201388817862608, + "0.9": 0.6228505674904834 + }, + "matched_frac_abs": { + "0.5": 0.6350911458333334, + "0.6": 0.57763671875, + "0.7": 0.4921061197916667, + "0.8": 0.3837890625, + "0.9": 0.21712239583333334 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.6314637660980225, + "dcs_abs": 0.6327425837516785, + "frac_sign_flipped": 0.032877604166666664, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.6281434297561646, + "0.6": 0.6306098103523254, + "0.7": 0.6314363479614258, + "0.8": 0.6314904689788818, + "0.9": 0.6314637660980225 + }, + "unique_frac_signed": { + "0.5": 0.989501953125, + "0.6": 0.9969889322916666, + "0.7": 0.999267578125, + "0.8": 0.9998779296875, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6215491505863611, + "0.6": 0.6287110015158153, + "0.7": 0.6309738701675087, + "0.8": 0.6314133827399928, + "0.9": 0.6314637660980225 + }, + "matched_frac_signed": { + "0.5": 0.6499837239583334, + "0.6": 0.5835774739583334, + "0.7": 0.4996744791666667, + "0.8": 0.389404296875, + "0.9": 0.21769205729166666 + }, + "matched_cos_abs": { + "0.5": 0.6300397515296936, + "0.6": 0.6318488717079163, + "0.7": 0.6320621967315674, + "0.8": 0.6315925717353821, + "0.9": 0.6319719552993774 + }, + "unique_frac_abs": { + "0.5": 0.958984375, + "0.6": 0.9706624348958334, + "0.7": 0.9756266276041666, + "0.8": 0.9788411458333334, + "0.9": 0.9829915364583334 + }, + "dedup_dcs_abs": { + "0.5": 0.6041982773458585, + "0.6": 0.613311964298191, + "0.7": 0.6166567094333004, + "0.8": 0.6182287966172832, + "0.9": 0.6212230833383122 + }, + "matched_frac_abs": { + "0.5": 0.6328938802083334, + "0.6": 0.5711263020833334, + "0.7": 0.4896647135416667, + "0.8": 0.3806966145833333, + "0.9": 0.21232096354166666 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 309, + "n_dense_std": 5.8594652770823155, + "l0_mean": 129, + "l0_std": 0.5092305101268828, + "recon_r2_mean": 0.9590603228740578, + "recon_r2_std": 0.008683364609472255, + "l1_per_token_mean": 110.39132902750283, + "l1_per_token_std": 0.16533168806228254, + "babel_mu1_signed_mean": 0.26738689343134564, + "babel_mu1_signed_std": 0.0012199759894576964, + "n_dup_signed_per_threshold": { + "0.5": 241.33333333333334, + "0.6": 60.0, + "0.7": 13.0, + "0.8": 3.3333333333333335, + "0.9": 0.0 + }, + "n_dup_signed_band": 19.083333333333332, + "n_dup_signed_band_std": 4.185789451624787, + "n_dup_clusters_signed_per_threshold": { + "0.5": 115.33333333333333, + "0.6": 27.333333333333332, + "0.7": 5.333333333333333, + "0.8": 1.6666666666666667, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 8.583333333333332, + "n_dup_clusters_signed_band_std": 1.8085445345175588, + "largest_dup_signed_per_threshold": { + "0.5": 7.0, + "0.6": 6.333333333333333, + "0.7": 4.333333333333333, + "0.8": 2.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 3.4166666666666665, + "largest_dup_signed_band_std": 0.7637626158259733, + "dcs_signed_mean": 0.6330980062484741, + "dcs_signed_std": 0.0014516298157111421, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.6234042709499287, + "0.6": 0.6308424599038618, + "0.7": 0.6327194277190655, + "0.8": 0.6329922960350006, + "0.9": 0.6330980062484741 + }, + "dedup_dcs_signed_band": 0.6324130474766005, + "dedup_dcs_signed_band_std": 0.0015665636009546844, + "matched_cos_signed_per_threshold": { + "0.5": 0.6298626661300659, + "0.6": 0.6325236360232035, + "0.7": 0.6331143379211426, + "0.8": 0.6330781777699789, + "0.9": 0.6330980062484741 + }, + "matched_cos_signed_band": 0.6329535394906998, + "matched_cos_signed_band_std": 0.0015073547645143405, + "unique_frac_signed_per_threshold": { + "0.5": 0.98974609375, + "0.6": 0.9973415798611112, + "0.7": 0.9993760850694445, + "0.8": 0.9998643663194445, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9991455078125001, + "unique_frac_signed_band_std": 9.703971368583687e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.6522081163194445, + "0.6": 0.5869683159722223, + "0.7": 0.5003526475694445, + "0.8": 0.392333984375, + "0.9": 0.22127278645833334 + }, + "matched_frac_signed_band": 0.42523193359375006, + "matched_frac_signed_band_std": 0.0023463058664287776, + "babel_mu1_abs_mean": 0.32881106932957965, + "babel_mu1_abs_std": 0.0012868300849916974, + "n_dup_abs_per_threshold": { + "0.5": 993.3333333333334, + "0.6": 716.0, + "0.7": 597.0, + "0.8": 518.6666666666666, + "0.9": 418.0 + }, + "n_dup_abs_band": 562.4166666666666, + "n_dup_abs_band_std": 6.256663114898654, + "n_dup_clusters_abs_per_threshold": { + "0.5": 490.6666666666667, + "0.6": 355.3333333333333, + "0.7": 297.3333333333333, + "0.8": 259.3333333333333, + "0.9": 209.0 + }, + "n_dup_clusters_abs_band": 280.25, + "n_dup_clusters_abs_band_std": 3.278719262151, + "largest_dup_abs_per_threshold": { + "0.5": 7.0, + "0.6": 6.333333333333333, + "0.7": 4.333333333333333, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 3.6666666666666665, + "largest_dup_abs_band_std": 0.7637626158259733, + "dcs_abs_mean": 0.6344483494758606, + "dcs_abs_std": 0.0015154685393691802, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6056873054379442, + "0.6": 0.6145901658843892, + "0.7": 0.6180519832252888, + "0.8": 0.6197605336831961, + "0.9": 0.6227052317085003 + }, + "dedup_dcs_abs_band": 0.6187769786253435, + "dedup_dcs_abs_band_std": 0.00126067605286934, + "matched_cos_abs_per_threshold": { + "0.5": 0.6315210064252218, + "0.6": 0.6331746379534403, + "0.7": 0.6335012118021647, + "0.8": 0.6331222852071127, + "0.9": 0.6334796746571859 + }, + "matched_cos_abs_band": 0.6333194524049759, + "matched_cos_abs_band_std": 0.00130556285728054, + "unique_frac_abs_per_threshold": { + "0.5": 0.9590928819444445, + "0.6": 0.9706488715277778, + "0.7": 0.9756130642361112, + "0.8": 0.9788953993055555, + "0.9": 0.9829915364583334 + }, + "unique_frac_abs_band": 0.9770372178819444, + "unique_frac_abs_band_std": 0.00012221151591620516, + "matched_frac_abs_per_threshold": { + "0.5": 0.6344401041666666, + "0.6": 0.573974609375, + "0.7": 0.48974609375, + "0.8": 0.38373480902777773, + "0.9": 0.21571180555555555 + }, + "matched_frac_abs_band": 0.4157918294270833, + "matched_frac_abs_band_std": 0.002144353047362962, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 12288 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed0/config.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f68b1fb974944b61099202c8efde7b2e80afeb45 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed0/lock.lock b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed0/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..7d8536d01a64f3e861e291daf72efe334ab9b63a --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.62628237304688, + "l0_per_token_0.001": 128.62628237304688, + "l0_per_token_0.01": 128.62628237304688, + "l0_per_token_0.05": 128.62628237304688, + "l0_per_token_0.1": 128.62628237304688, + "l0_per_token_0.5": 71.61629750976563, + "l0_per_token_1.0": 24.904558740234375, + "l1_per_token": 110.25429557056879, + "r2_score": 0.9543583405070089, + "l2_loss": 0.03038689273130135 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed1/config.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7bbdb76129185355c9135442b24bb62c1df7a497 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed1/lock.lock b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed1/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..2fc35103d78bda073c55e034ecd018bca1283698 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 129.00770043945312, + "l0_per_token_0.001": 129.00770043945312, + "l0_per_token_0.01": 129.00770043945312, + "l0_per_token_0.05": 129.00770043945312, + "l0_per_token_0.1": 129.00770043945312, + "l0_per_token_0.5": 71.67238862304687, + "l0_per_token_1.0": 25.17853291015625, + "l1_per_token": 110.57495848696888, + "r2_score": 0.9537419371398657, + "l2_loss": 0.030797276210006115 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed2/config.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..65ad07e6db1c726a7a35612ab887cd3f3b403c4c --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed2/lock.lock b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed2/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..90a877bd40fc62b202f27945386e9dfa78d84df6 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 127.99916665039062, + "l0_per_token_0.001": 127.99916665039062, + "l0_per_token_0.01": 127.99916665039062, + "l0_per_token_0.05": 127.99916665039062, + "l0_per_token_0.1": 127.99916665039062, + "l0_per_token_0.5": 71.8047177734375, + "l0_per_token_1.0": 25.102107421875, + "l1_per_token": 110.34473302497086, + "r2_score": 0.969080690975299, + "l2_loss": 0.030440846090040087 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/code_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..c0ea232062a70a243c354651c11ab5aa69acaf4e --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 32.097076666666666, + "recon_r2": 0.9141570895304405, + "l1_per_token": 58.28440006958574 + }, + "seed1": { + "l0": 32.478675, + "recon_r2": 0.9138857128671511, + "l1_per_token": 58.51861732383281 + }, + "seed2": { + "l0": 32.14297833333333, + "recon_r2": 0.9141882463847553, + "l1_per_token": 58.16773558372269 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7148869301446154, + "mcc_abs": 0.7148991332696465, + "act_iou": 0.4072820703469975, + "act_iou_n_pairs": 12288, + "input_r2": 0.9680945371875234, + "matched_mcc_signed": { + "0.6": 0.835930421852289, + "0.7": 0.8667161435504823, + "0.8": 0.8959182880863512, + "0.9": 0.928033566804409 + }, + "matched_mcc_abs": { + "0.6": 0.8348793084512307, + "0.7": 0.8664840434913818, + "0.8": 0.8958044407998295, + "0.9": 0.9280251687741714 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7169640103757814, + "mcc_abs": 0.716972491718083, + "act_iou": 0.4092259615607889, + "act_iou_n_pairs": 12288, + "input_r2": 0.9697843886617002, + "matched_mcc_signed": { + "0.6": 0.8348537666244693, + "0.7": 0.8663806808552532, + "0.8": 0.8958246993211095, + "0.9": 0.930200857503117 + }, + "matched_mcc_abs": { + "0.6": 0.8340004539093876, + "0.7": 0.8660388413020383, + "0.8": 0.895692922281692, + "0.9": 0.930181995463511 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.716574307187253, + "mcc_abs": 0.7165846895416578, + "act_iou": 0.40716243209210273, + "act_iou_n_pairs": 12288, + "input_r2": 0.9684869661533552, + "matched_mcc_signed": { + "0.6": 0.8345879505559878, + "0.7": 0.865994809398524, + "0.8": 0.8957127861243994, + "0.9": 0.928545635333421 + }, + "matched_mcc_abs": { + "0.6": 0.8345846457343309, + "0.7": 0.8659199923433918, + "0.8": 0.895599133604797, + "0.9": 0.9284550578186145 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 32.239576666666665, + "l0_std": 0.208333267873325, + "recon_r2_mean": 0.9140770162607823, + "recon_r2_std": 0.00016640441311394514, + "l1_per_token_mean": 58.32358432571374, + "l1_per_token_std": 0.1786926224887442, + "input_r2_mean": 0.9687886306675262, + "input_r2_std": 0.0008843927919917758, + "mcc_signed_mean": 0.7161417492358831, + "mcc_signed_std": 0.0011040359403774389, + "mcc_abs_mean": 0.7161521048431291, + "mcc_abs_std": 0.0011022934976568635, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8351240463442487, + "0.7": 0.8663638779347531, + "0.8": 0.8958185911772868, + "0.9": 0.9289266865469824 + }, + "matched_mcc_signed_band": 0.8815583005008178, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8344881360316497, + "0.7": 0.8661476257122707, + "0.8": 0.8956988322287728, + "0.9": 0.9288874073520988 + }, + "matched_mcc_abs_band": 0.881305500331198, + "act_iou_mean": 0.40789015466662976, + "act_iou_std": 0.001158388264732932, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/dict_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..ffef4026bc069be126b474f6bbc1b632573c9a89 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 32.0096005859375, + "recon_r2": 0.9145350044072166, + "l1_per_token": 58.11472164700112, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 22, + "dense_rate": 0.0017903645833333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1336, + "0.6": 353, + "0.7": 64, + "0.8": 10, + "0.9": 2 + }, + "n_dup_clusters_signed": { + "0.5": 633, + "0.6": 163, + "0.7": 29, + "0.8": 4, + "0.9": 1 + }, + "n_dup_removed_signed": { + "0.5": 703, + "0.6": 190, + "0.7": 35, + "0.8": 6, + "0.9": 1 + }, + "largest_dup_signed": { + "0.5": 22, + "0.6": 17, + "0.7": 5, + "0.8": 4, + "0.9": 2 + }, + "babel_mu1_signed": 0.3818216621875763, + "n_dup_abs": { + "0.5": 1523, + "0.6": 513, + "0.7": 186, + "0.8": 100, + "0.9": 60 + }, + "n_dup_clusters_abs": { + "0.5": 726, + "0.6": 243, + "0.7": 90, + "0.8": 49, + "0.9": 30 + }, + "n_dup_removed_abs": { + "0.5": 797, + "0.6": 270, + "0.7": 96, + "0.8": 51, + "0.9": 30 + }, + "largest_dup_abs": { + "0.5": 22, + "0.6": 17, + "0.7": 5, + "0.8": 4, + "0.9": 2 + }, + "babel_mu1_abs": 0.3934354782104492, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 32.35680791015625, + "recon_r2": 0.9143079087538538, + "l1_per_token": 58.32559807712143, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 22, + "dense_rate": 0.0017903645833333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1401, + "0.6": 378, + "0.7": 68, + "0.8": 8, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 663, + "0.6": 178, + "0.7": 34, + "0.8": 4, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 738, + "0.6": 200, + "0.7": 34, + "0.8": 4, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 11, + "0.6": 9, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.3824748992919922, + "n_dup_abs": { + "0.5": 1613, + "0.6": 540, + "0.7": 202, + "0.8": 120, + "0.9": 68 + }, + "n_dup_clusters_abs": { + "0.5": 769, + "0.6": 259, + "0.7": 101, + "0.8": 60, + "0.9": 34 + }, + "n_dup_removed_abs": { + "0.5": 844, + "0.6": 281, + "0.7": 101, + "0.8": 60, + "0.9": 34 + }, + "largest_dup_abs": { + "0.5": 11, + "0.6": 9, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.39461302757263184, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 32.03309653320313, + "recon_r2": 0.9422383526800548, + "l1_per_token": 57.987864474431206, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 21, + "dense_rate": 0.001708984375, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1417, + "0.6": 347, + "0.7": 56, + "0.8": 8, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 676, + "0.6": 165, + "0.7": 26, + "0.8": 4, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 741, + "0.6": 182, + "0.7": 30, + "0.8": 4, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 16, + "0.6": 10, + "0.7": 4, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.3819083273410797, + "n_dup_abs": { + "0.5": 1608, + "0.6": 505, + "0.7": 178, + "0.8": 108, + "0.9": 62 + }, + "n_dup_clusters_abs": { + "0.5": 771, + "0.6": 244, + "0.7": 87, + "0.8": 54, + "0.9": 31 + }, + "n_dup_removed_abs": { + "0.5": 837, + "0.6": 261, + "0.7": 91, + "0.8": 54, + "0.9": 31 + }, + "largest_dup_abs": { + "0.5": 16, + "0.6": 10, + "0.7": 4, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3939823806285858, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.7876917719841003, + "dcs_abs": 0.7886753082275391, + "frac_sign_flipped": 0.011962890625, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7780671715736389, + "0.6": 0.7852004170417786, + "0.7": 0.787301242351532, + "0.8": 0.7876412272453308, + "0.9": 0.7877033948898315 + }, + "unique_frac_signed": { + "0.5": 0.9413655598958334, + "0.6": 0.984130859375, + "0.7": 0.9971923828125, + "0.8": 0.9995930989583334, + "0.9": 0.9999593098958334 + }, + "dedup_dcs_signed": { + "0.5": 0.732445638604986, + "0.6": 0.7727399612049339, + "0.7": 0.7850908018517657, + "0.8": 0.7873207352095051, + "0.9": 0.7876713431566411 + }, + "matched_frac_signed": { + "0.5": 0.7857259114583334, + "0.6": 0.780029296875, + "0.7": 0.7316080729166666, + "0.8": 0.6433919270833334, + "0.9": 0.4889322916666667 + }, + "matched_cos_abs": { + "0.5": 0.7800617814064026, + "0.6": 0.7867946028709412, + "0.7": 0.7888025641441345, + "0.8": 0.7890578508377075, + "0.9": 0.7890185117721558 + }, + "unique_frac_abs": { + "0.5": 0.9332275390625, + "0.6": 0.9775797526041666, + "0.7": 0.9919840494791666, + "0.8": 0.9954833984375, + "0.9": 0.9973958333333334 + }, + "dedup_dcs_abs": { + "0.5": 0.7279751365786069, + "0.6": 0.7691544732248682, + "0.7": 0.7824795618192486, + "0.8": 0.785493990915711, + "0.9": 0.7869637760644158 + }, + "matched_frac_abs": { + "0.5": 0.7806803385416666, + "0.6": 0.7777506510416666, + "0.7": 0.7293294270833334, + "0.8": 0.642333984375, + "0.9": 0.4883626302083333 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.7893560528755188, + "dcs_abs": 0.790317952632904, + "frac_sign_flipped": 0.013020833333333334, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7787999510765076, + "0.6": 0.7867252826690674, + "0.7": 0.7891610264778137, + "0.8": 0.7894879579544067, + "0.9": 0.7894057035446167 + }, + "unique_frac_signed": { + "0.5": 0.9412434895833334, + "0.6": 0.98486328125, + "0.7": 0.9973551432291666, + "0.8": 0.9995930989583334, + "0.9": 0.9999593098958334 + }, + "dedup_dcs_signed": { + "0.5": 0.7330403836385813, + "0.6": 0.7748168433317915, + "0.7": 0.787073808593656, + "0.8": 0.7891667144819319, + "0.9": 0.7893735825443098 + }, + "matched_frac_signed": { + "0.5": 0.78125, + "0.6": 0.7854817708333334, + "0.7": 0.7353515625, + "0.8": 0.6490071614583334, + "0.9": 0.490478515625 + }, + "matched_cos_abs": { + "0.5": 0.7806100249290466, + "0.6": 0.7881104946136475, + "0.7": 0.7902831435203552, + "0.8": 0.790547788143158, + "0.9": 0.7905182838439941 + }, + "unique_frac_abs": { + "0.5": 0.9335123697916666, + "0.6": 0.9783935546875, + "0.7": 0.9923909505208334, + "0.8": 0.9957275390625, + "0.9": 0.9975179036458334 + }, + "dedup_dcs_abs": { + "0.5": 0.7287091142546462, + "0.6": 0.7710822283115704, + "0.7": 0.7842698399787575, + "0.8": 0.7871702035990893, + "0.9": 0.788556141293763 + }, + "matched_frac_abs": { + "0.5": 0.7770182291666666, + "0.6": 0.7823079427083334, + "0.7": 0.7333170572916666, + "0.8": 0.647705078125, + "0.9": 0.4893391927083333 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.7889164090156555, + "dcs_abs": 0.7899033427238464, + "frac_sign_flipped": 0.011393229166666666, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7766858339309692, + "0.6": 0.7860124111175537, + "0.7": 0.7885205149650574, + "0.8": 0.7888569235801697, + "0.9": 0.7889164090156555 + }, + "unique_frac_signed": { + "0.5": 0.9398193359375, + "0.6": 0.9844563802083334, + "0.7": 0.9973958333333334, + "0.8": 0.9996744791666666, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7299443646770669, + "0.6": 0.7737949330476113, + "0.7": 0.7864670761240026, + "0.8": 0.788600134217025, + "0.9": 0.7889164090156555 + }, + "matched_frac_signed": { + "0.5": 0.78125, + "0.6": 0.7815755208333334, + "0.7": 0.7315266927083334, + "0.8": 0.6429036458333334, + "0.9": 0.48876953125 + }, + "matched_cos_abs": { + "0.5": 0.7781708240509033, + "0.6": 0.7873791456222534, + "0.7": 0.7897810339927673, + "0.8": 0.7901405096054077, + "0.9": 0.7900776267051697 + }, + "unique_frac_abs": { + "0.5": 0.9315999348958334, + "0.6": 0.9779459635416666, + "0.7": 0.9921875, + "0.8": 0.995361328125, + "0.9": 0.9973551432291666 + }, + "dedup_dcs_abs": { + "0.5": 0.7249438890236586, + "0.6": 0.7700142572381689, + "0.7": 0.7836108696646988, + "0.8": 0.7864753070462029, + "0.9": 0.7879879845446945 + }, + "matched_frac_abs": { + "0.5": 0.775146484375, + "0.6": 0.7782389322916666, + "0.7": 0.72900390625, + "0.8": 0.6416829427083334, + "0.9": 0.4879557291666667 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 22, + "n_dense_std": 0.5773502691896258, + "l0_mean": 32, + "l0_std": 0.19403351991255666, + "recon_r2_mean": 0.9236937552803751, + "recon_r2_std": 0.016060493848138707, + "l1_per_token_mean": 58.14272806618459, + "l1_per_token_std": 0.1705997251848286, + "babel_mu1_signed_mean": 0.3820682962735494, + "babel_mu1_signed_std": 0.0003547847561029553, + "n_dup_signed_per_threshold": { + "0.5": 1384.6666666666667, + "0.6": 359.3333333333333, + "0.7": 62.666666666666664, + "0.8": 8.666666666666666, + "0.9": 0.6666666666666666 + }, + "n_dup_signed_band": 107.83333333333334, + "n_dup_signed_band_std": 5.3986881122485055, + "n_dup_clusters_signed_per_threshold": { + "0.5": 657.3333333333334, + "0.6": 168.66666666666666, + "0.7": 29.666666666666668, + "0.8": 4.0, + "0.9": 0.3333333333333333 + }, + "n_dup_clusters_signed_band": 50.666666666666664, + "n_dup_clusters_signed_band_std": 2.897556441785618, + "largest_dup_signed_per_threshold": { + "0.5": 16.333333333333332, + "0.6": 12.0, + "0.7": 3.6666666666666665, + "0.8": 2.6666666666666665, + "0.9": 1.3333333333333333 + }, + "largest_dup_signed_band": 4.916666666666666, + "largest_dup_signed_band_std": 1.8427786989579984, + "dcs_signed_mean": 0.7886547446250916, + "dcs_signed_std": 0.0008624435699522966, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7318101289735447, + "0.6": 0.7737839125281122, + "0.7": 0.7862105621898081, + "0.8": 0.7883625279694874, + "0.9": 0.7886537782388688 + }, + "dedup_dcs_signed_band": 0.7842526952315692, + "dedup_dcs_signed_band_std": 0.0009654315687962092, + "matched_cos_signed_per_threshold": { + "0.5": 0.7778509855270386, + "0.6": 0.7859793702761332, + "0.7": 0.7883275945981344, + "0.8": 0.7886620362599691, + "0.9": 0.7886751691500345 + }, + "matched_cos_signed_band": 0.7879110425710678, + "matched_cos_signed_band_std": 0.0008784852722002345, + "unique_frac_signed_per_threshold": { + "0.5": 0.9408094618055557, + "0.6": 0.9844835069444445, + "0.7": 0.997314453125, + "0.8": 0.9996202256944445, + "0.9": 0.999972873263889 + }, + "unique_frac_signed_band": 0.9953477647569445, + "unique_frac_signed_band_std": 0.00011568686326245756, + "matched_frac_signed_per_threshold": { + "0.5": 0.7827419704861112, + "0.6": 0.7823621961805557, + "0.7": 0.7328287760416666, + "0.8": 0.6451009114583334, + "0.9": 0.4893934461805556 + }, + "matched_frac_signed_band": 0.6624213324652779, + "matched_frac_signed_band_std": 0.0023045056421594204, + "babel_mu1_abs_mean": 0.3940102954705556, + "babel_mu1_abs_std": 0.0005892707814713213, + "n_dup_abs_per_threshold": { + "0.5": 1581.3333333333333, + "0.6": 519.3333333333334, + "0.7": 188.66666666666666, + "0.8": 109.33333333333333, + "0.9": 63.333333333333336 + }, + "n_dup_abs_band": 220.16666666666669, + "n_dup_abs_band_std": 10.70727945527403, + "n_dup_clusters_abs_per_threshold": { + "0.5": 755.3333333333334, + "0.6": 248.66666666666666, + "0.7": 92.66666666666667, + "0.8": 54.333333333333336, + "0.9": 31.666666666666668 + }, + "n_dup_clusters_abs_band": 106.83333333333333, + "n_dup_clusters_abs_band_std": 5.795112883571236, + "largest_dup_abs_per_threshold": { + "0.5": 16.333333333333332, + "0.6": 12.0, + "0.7": 3.6666666666666665, + "0.8": 2.6666666666666665, + "0.9": 2.0 + }, + "largest_dup_abs_band": 5.083333333333333, + "largest_dup_abs_band_std": 1.7017148213885114, + "dcs_abs_mean": 0.7896322011947632, + "dcs_abs_std": 0.000854229744983414, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.727209379952304, + "0.6": 0.770083652924869, + "0.7": 0.7834534238209017, + "0.8": 0.7863798338536677, + "0.9": 0.7878359673009577 + }, + "dedup_dcs_abs_band": 0.781938219475099, + "dedup_dcs_abs_band_std": 0.0008763427015476509, + "matched_cos_abs_per_threshold": { + "0.5": 0.7796142101287842, + "0.6": 0.787428081035614, + "0.7": 0.7896222472190857, + "0.8": 0.7899153828620911, + "0.9": 0.7898714741071066 + }, + "matched_cos_abs_band": 0.7892092963059744, + "matched_cos_abs_band_std": 0.000732699973170274, + "unique_frac_abs_per_threshold": { + "0.5": 0.9327799479166666, + "0.6": 0.9779730902777777, + "0.7": 0.9921875, + "0.8": 0.9955240885416666, + "0.9": 0.9974229600694445 + }, + "unique_frac_abs_band": 0.9907769097222222, + "unique_frac_abs_band_std": 0.00020606166726675756, + "matched_frac_abs_per_threshold": { + "0.5": 0.777615017361111, + "0.6": 0.7794325086805555, + "0.7": 0.7305501302083334, + "0.8": 0.6439073350694445, + "0.9": 0.4885525173611111 + }, + "matched_frac_abs_band": 0.6606106228298612, + "matched_frac_abs_band_std": 0.00221698841693387, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 12288 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed0/config.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1471f6d35c7bc8f1a7d2c3f7010e993d4c4e69ce --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed0/lock.lock b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed0/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..200143f8c5efd7d8267056cfa4bb66cf2484a7fe --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.0096005859375, + "l0_per_token_0.001": 32.0096005859375, + "l0_per_token_0.01": 32.0096005859375, + "l0_per_token_0.05": 32.0096005859375, + "l0_per_token_0.1": 32.0096005859375, + "l0_per_token_0.5": 32.0096005859375, + "l0_per_token_1.0": 17.600351318359376, + "l1_per_token": 58.11472164700112, + "r2_score": 0.9145350044072166, + "l2_loss": 0.0569001145490307 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed1/config.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..113dc9e5733aeadd53c79f20243ec2c98e11f200 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed1/lock.lock b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed1/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b06a98767e4e4cb18fbbab585a0e2fb5d0c1bd64 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.35680791015625, + "l0_per_token_0.001": 32.35680791015625, + "l0_per_token_0.01": 32.35680791015625, + "l0_per_token_0.05": 32.35680791015625, + "l0_per_token_0.1": 32.35680791015625, + "l0_per_token_0.5": 32.35680791015625, + "l0_per_token_1.0": 17.656039208984375, + "l1_per_token": 58.32559807712143, + "r2_score": 0.9143079087538538, + "l2_loss": 0.0570513082465241 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed2/config.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f96b94c5cdf87a80891a0447037ec702c16750c5 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed2/lock.lock b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed2/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b6571b626b7c8f32b462ae9181bd7b1734abdf17 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.03309653320313, + "l0_per_token_0.001": 32.03309653320313, + "l0_per_token_0.01": 32.03309653320313, + "l0_per_token_0.05": 32.03309653320313, + "l0_per_token_0.1": 32.03309653320313, + "l0_per_token_0.5": 32.03309653320313, + "l0_per_token_1.0": 17.59882373046875, + "l1_per_token": 57.987864474431206, + "r2_score": 0.9422383526800548, + "l2_loss": 0.05686781080938566 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/code_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..8e751a42007b39df846a8e5238112c1e376a7e43 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 64.48569833333333, + "recon_r2": 0.9344001150220504, + "l1_per_token": 77.65846787420764 + }, + "seed1": { + "l0": 64.755275, + "recon_r2": 0.9341566907093881, + "l1_per_token": 77.66975336423148 + }, + "seed2": { + "l0": 64.19316333333333, + "recon_r2": 0.9342717706693233, + "l1_per_token": 77.29708878281842 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6741245279653785, + "mcc_abs": 0.674132111567857, + "act_iou": 0.34825414556843975, + "act_iou_n_pairs": 12288, + "input_r2": 0.9693088127235118, + "matched_mcc_signed": { + "0.6": 0.8388153626633673, + "0.7": 0.868469923611084, + "0.8": 0.9002290519034732, + "0.9": 0.9314748397080376 + }, + "matched_mcc_abs": { + "0.6": 0.8390370441585818, + "0.7": 0.8684322374146111, + "0.8": 0.9002553735966271, + "0.9": 0.9314974848522927 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6770412687658641, + "mcc_abs": 0.6770498955802421, + "act_iou": 0.35099353214444934, + "act_iou_n_pairs": 12288, + "input_r2": 0.9696832601513978, + "matched_mcc_signed": { + "0.6": 0.8385268903660195, + "0.7": 0.8692595122146864, + "0.8": 0.9006092151138925, + "0.9": 0.9342435710080322 + }, + "matched_mcc_abs": { + "0.6": 0.8392140803626449, + "0.7": 0.8693632538967202, + "0.8": 0.90039329592944, + "0.9": 0.9340329767202393 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6749820409341228, + "mcc_abs": 0.6749916872665572, + "act_iou": 0.3500190188800197, + "act_iou_n_pairs": 12288, + "input_r2": 0.9696497739750458, + "matched_mcc_signed": { + "0.6": 0.8378752318911785, + "0.7": 0.8691048549055447, + "0.8": 0.8996169541807075, + "0.9": 0.9325544604038507 + }, + "matched_mcc_abs": { + "0.6": 0.8381515459881593, + "0.7": 0.8693008614635855, + "0.8": 0.899722071022894, + "0.9": 0.9325755922209806 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 64.47804555555555, + "l0_std": 0.2811339630967441, + "recon_r2_mean": 0.9342761921335873, + "recon_r2_std": 0.00012177237374897666, + "l1_per_token_mean": 77.54177000708586, + "l1_per_token_std": 0.21197527380103887, + "input_r2_mean": 0.9695472822833184, + "input_r2_std": 0.00020719828478409522, + "mcc_signed_mean": 0.6753826125551218, + "mcc_signed_std": 0.0014990621874914753, + "mcc_abs_mean": 0.6753912314715521, + "mcc_abs_std": 0.0014993640533710376, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8384058283068551, + "0.7": 0.868944763577105, + "0.8": 0.9001517403993576, + "0.9": 0.9327576237066402 + }, + "matched_mcc_signed_band": 0.8850649889974894, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8388008901697953, + "0.7": 0.869032117591639, + "0.8": 0.9001235801829871, + "0.9": 0.9327020179311708 + }, + "matched_mcc_abs_band": 0.885164651468898, + "act_iou_mean": 0.3497555655309696, + "act_iou_std": 0.0013885659701827853, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/dict_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..a8c93d6bb00367858e6b5bf0a87d086a901f2414 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 64.30206723632813, + "recon_r2": 0.9346842268803025, + "l1_per_token": 77.41030274099167, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 73, + "dense_rate": 0.005940755208333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 540, + "0.6": 121, + "0.7": 32, + "0.8": 11, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 256, + "0.6": 55, + "0.7": 12, + "0.8": 4, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 284, + "0.6": 66, + "0.7": 20, + "0.8": 7, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 12, + "0.6": 11, + "0.7": 9, + "0.8": 5, + "0.9": 1 + }, + "babel_mu1_signed": 0.3165056109428406, + "n_dup_abs": { + "0.5": 920, + "0.6": 430, + "0.7": 289, + "0.8": 229, + "0.9": 164 + }, + "n_dup_clusters_abs": { + "0.5": 444, + "0.6": 208, + "0.7": 140, + "0.8": 113, + "0.9": 82 + }, + "n_dup_removed_abs": { + "0.5": 476, + "0.6": 222, + "0.7": 149, + "0.8": 116, + "0.9": 82 + }, + "largest_dup_abs": { + "0.5": 13, + "0.6": 12, + "0.7": 9, + "0.8": 5, + "0.9": 2 + }, + "babel_mu1_abs": 0.3434722423553467, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 65, + "l0_raw": 64.53487338867187, + "recon_r2": 0.9344907847309781, + "l1_per_token": 77.39352973653504, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 72, + "dense_rate": 0.005859375, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 540, + "0.6": 126, + "0.7": 31, + "0.8": 13, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 257, + "0.6": 58, + "0.7": 13, + "0.8": 5, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 283, + "0.6": 68, + "0.7": 18, + "0.8": 8, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 9, + "0.6": 9, + "0.7": 6, + "0.8": 5, + "0.9": 1 + }, + "babel_mu1_signed": 0.31616875529289246, + "n_dup_abs": { + "0.5": 921, + "0.6": 471, + "0.7": 331, + "0.8": 261, + "0.9": 176 + }, + "n_dup_clusters_abs": { + "0.5": 446, + "0.6": 230, + "0.7": 163, + "0.8": 129, + "0.9": 88 + }, + "n_dup_removed_abs": { + "0.5": 475, + "0.6": 241, + "0.7": 168, + "0.8": 132, + "0.9": 88 + }, + "largest_dup_abs": { + "0.5": 9, + "0.6": 9, + "0.7": 6, + "0.8": 5, + "0.9": 2 + }, + "babel_mu1_abs": 0.3442070782184601, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 63.99384736328125, + "recon_r2": 0.9557593676449077, + "l1_per_token": 77.04586503847759, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 74, + "dense_rate": 0.006022135416666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 543, + "0.6": 131, + "0.7": 30, + "0.8": 13, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 261, + "0.6": 60, + "0.7": 11, + "0.8": 4, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 282, + "0.6": 71, + "0.7": 19, + "0.8": 9, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 11, + "0.6": 11, + "0.7": 9, + "0.8": 6, + "0.9": 1 + }, + "babel_mu1_signed": 0.31703412532806396, + "n_dup_abs": { + "0.5": 953, + "0.6": 475, + "0.7": 314, + "0.8": 243, + "0.9": 170 + }, + "n_dup_clusters_abs": { + "0.5": 464, + "0.6": 231, + "0.7": 153, + "0.8": 119, + "0.9": 85 + }, + "n_dup_removed_abs": { + "0.5": 489, + "0.6": 244, + "0.7": 161, + "0.8": 124, + "0.9": 85 + }, + "largest_dup_abs": { + "0.5": 12, + "0.6": 12, + "0.7": 9, + "0.8": 6, + "0.9": 2 + }, + "babel_mu1_abs": 0.34514322876930237, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.7282670140266418, + "dcs_abs": 0.7293965816497803, + "frac_sign_flipped": 0.019287109375, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7224281430244446, + "0.6": 0.727152407169342, + "0.7": 0.7281764149665833, + "0.8": 0.7282457947731018, + "0.9": 0.7282670140266418 + }, + "unique_frac_signed": { + "0.5": 0.9769287109375, + "0.6": 0.9945475260416666, + "0.7": 0.9984537760416666, + "0.8": 0.9993896484375, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7057607945098425, + "0.6": 0.7231876276055118, + "0.7": 0.7270504911478686, + "0.8": 0.727801308814378, + "0.9": 0.7282670140266418 + }, + "matched_frac_signed": { + "0.5": 0.7574055989583334, + "0.6": 0.71240234375, + "0.7": 0.6444498697916666, + "0.8": 0.53662109375, + "0.9": 0.3719075520833333 + }, + "matched_cos_abs": { + "0.5": 0.7242125868797302, + "0.6": 0.7293079495429993, + "0.7": 0.7300531268119812, + "0.8": 0.7299748659133911, + "0.9": 0.7297325134277344 + }, + "unique_frac_abs": { + "0.5": 0.9613037109375, + "0.6": 0.9811604817708334, + "0.7": 0.9871012369791666, + "0.8": 0.9899088541666666, + "0.9": 0.9930826822916666 + }, + "dedup_dcs_abs": { + "0.5": 0.6961882472751313, + "0.6": 0.7155681391329078, + "0.7": 0.7206363445366151, + "0.8": 0.7226085830867911, + "0.9": 0.7246847217902541 + }, + "matched_frac_abs": { + "0.5": 0.7467447916666666, + "0.6": 0.705322265625, + "0.7": 0.6402994791666666, + "0.8": 0.5335286458333334, + "0.9": 0.36962890625 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.7297753691673279, + "dcs_abs": 0.7309150695800781, + "frac_sign_flipped": 0.020589192708333332, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7237417697906494, + "0.6": 0.7286899089813232, + "0.7": 0.7297379970550537, + "0.8": 0.7296972870826721, + "0.9": 0.7297753691673279 + }, + "unique_frac_signed": { + "0.5": 0.9769694010416666, + "0.6": 0.9944254557291666, + "0.7": 0.9984130859375, + "0.8": 0.9993489583333334, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7070735633412065, + "0.6": 0.7246277948239973, + "0.7": 0.7285799655655865, + "0.8": 0.7292222237447277, + "0.9": 0.7297753691673279 + }, + "matched_frac_signed": { + "0.5": 0.7569986979166666, + "0.6": 0.7189127604166666, + "0.7": 0.6478678385416666, + "0.8": 0.5419921875, + "0.9": 0.3724772135416667 + }, + "matched_cos_abs": { + "0.5": 0.726472795009613, + "0.6": 0.7308323979377747, + "0.7": 0.7316621541976929, + "0.8": 0.7313475012779236, + "0.9": 0.7314083576202393 + }, + "unique_frac_abs": { + "0.5": 0.9607340494791666, + "0.6": 0.9810384114583334, + "0.7": 0.9873860677083334, + "0.8": 0.990234375, + "0.9": 0.9932047526041666 + }, + "dedup_dcs_abs": { + "0.5": 0.697947150186034, + "0.6": 0.716974654715159, + "0.7": 0.7224330173242682, + "0.8": 0.7242054358357564, + "0.9": 0.7264382568828296 + }, + "matched_frac_abs": { + "0.5": 0.74658203125, + "0.6": 0.7106119791666666, + "0.7": 0.6437174479166666, + "0.8": 0.5389811197916666, + "0.9": 0.3705240885416667 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.7292006611824036, + "dcs_abs": 0.7303452491760254, + "frac_sign_flipped": 0.020751953125, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.7232844233512878, + "0.6": 0.7278395295143127, + "0.7": 0.7288520932197571, + "0.8": 0.7291439771652222, + "0.9": 0.7292006611824036 + }, + "unique_frac_signed": { + "0.5": 0.9770100911458334, + "0.6": 0.9943440755208334, + "0.7": 0.9984944661458334, + "0.8": 0.9993082682291666, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7066561803828033, + "0.6": 0.7237229241024277, + "0.7": 0.7277547817187345, + "0.8": 0.7286396051107051, + "0.9": 0.7292006611824036 + }, + "matched_frac_signed": { + "0.5": 0.7578938802083334, + "0.6": 0.7177734375, + "0.7": 0.64453125, + "0.8": 0.542724609375, + "0.9": 0.371337890625 + }, + "matched_cos_abs": { + "0.5": 0.7253309488296509, + "0.6": 0.7293285727500916, + "0.7": 0.7303219437599182, + "0.8": 0.7304819226264954, + "0.9": 0.7305492758750916 + }, + "unique_frac_abs": { + "0.5": 0.9607747395833334, + "0.6": 0.9802652994791666, + "0.7": 0.9866129557291666, + "0.8": 0.9895833333333334, + "0.9": 0.9929606119791666 + }, + "dedup_dcs_abs": { + "0.5": 0.69687965347354, + "0.6": 0.7149354917855817, + "0.7": 0.7205450915668431, + "0.8": 0.7228727359324694, + "0.9": 0.7254066560538679 + }, + "matched_frac_abs": { + "0.5": 0.747314453125, + "0.6": 0.7091471354166666, + "0.7": 0.6385904947916666, + "0.8": 0.538818359375, + "0.9": 0.3688151041666667 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 73, + "n_dense_std": 1.0, + "l0_mean": 64, + "l0_std": 0.2713875933236773, + "recon_r2_mean": 0.9416447930853961, + "recon_r2_std": 0.012223962786809029, + "l1_per_token_mean": 77.28323250533477, + "l1_per_token_std": 0.20573725759149974, + "babel_mu1_signed_mean": 0.3165694971879323, + "babel_mu1_signed_std": 0.0004362079935960912, + "n_dup_signed_per_threshold": { + "0.5": 541.0, + "0.6": 126.0, + "0.7": 31.0, + "0.8": 12.333333333333334, + "0.9": 0.0 + }, + "n_dup_signed_band": 42.333333333333336, + "n_dup_signed_band_std": 1.2583057392117918, + "n_dup_clusters_signed_per_threshold": { + "0.5": 258.0, + "0.6": 57.666666666666664, + "0.7": 12.0, + "0.8": 4.333333333333333, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 18.499999999999996, + "n_dup_clusters_signed_band_std": 0.6614378277661477, + "largest_dup_signed_per_threshold": { + "0.5": 10.666666666666666, + "0.6": 10.333333333333334, + "0.7": 8.0, + "0.8": 5.333333333333333, + "0.9": 1.0 + }, + "largest_dup_signed_band": 6.166666666666667, + "largest_dup_signed_band_std": 0.8036375634160796, + "dcs_signed_mean": 0.7290810147921244, + "dcs_signed_std": 0.0007612622751973314, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7064968460779507, + "0.6": 0.7238461155106455, + "0.7": 0.7277950794773965, + "0.8": 0.7285543792232702, + "0.9": 0.7290810147921244 + }, + "dedup_dcs_signed_band": 0.7273191472508591, + "dedup_dcs_signed_band_std": 0.0007374183960720544, + "matched_cos_signed_per_threshold": { + "0.5": 0.723151445388794, + "0.6": 0.7278939485549927, + "0.7": 0.728922168413798, + "0.8": 0.7290290196736654, + "0.9": 0.7290810147921244 + }, + "matched_cos_signed_band": 0.7287315378586451, + "matched_cos_signed_band_std": 0.0007577415200341631, + "unique_frac_signed_per_threshold": { + "0.5": 0.9769694010416666, + "0.6": 0.9944390190972222, + "0.7": 0.9984537760416666, + "0.8": 0.9993489583333334, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9980604383680556, + "unique_frac_signed_band_std": 3.2700096167590827e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.7574327256944445, + "0.6": 0.7163628472222222, + "0.7": 0.6456163194444444, + "0.8": 0.5404459635416666, + "0.9": 0.3719075520833333 + }, + "matched_frac_signed_band": 0.5685831705729166, + "matched_frac_signed_band_std": 0.0020319604853757914, + "babel_mu1_abs_mean": 0.3442741831143697, + "babel_mu1_abs_std": 0.0008375119098839158, + "n_dup_abs_per_threshold": { + "0.5": 931.3333333333334, + "0.6": 458.6666666666667, + "0.7": 311.3333333333333, + "0.8": 244.33333333333334, + "0.9": 170.0 + }, + "n_dup_abs_band": 296.08333333333337, + "n_dup_abs_band_std": 16.329293718141436, + "n_dup_clusters_abs_per_threshold": { + "0.5": 451.3333333333333, + "0.6": 223.0, + "0.7": 152.0, + "0.8": 120.33333333333333, + "0.9": 85.0 + }, + "n_dup_clusters_abs_band": 145.08333333333331, + "n_dup_clusters_abs_band_std": 8.537905676062095, + "largest_dup_abs_per_threshold": { + "0.5": 11.333333333333334, + "0.6": 11.0, + "0.7": 8.0, + "0.8": 5.333333333333333, + "0.9": 2.0 + }, + "largest_dup_abs_band": 6.583333333333333, + "largest_dup_abs_band_std": 0.9464847243000456, + "dcs_abs_mean": 0.7302189668019613, + "dcs_abs_std": 0.0007670800656479853, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.697005016978235, + "0.6": 0.7158260952112162, + "0.7": 0.7212048178092422, + "0.8": 0.7232289182850056, + "0.9": 0.7255098782423172 + }, + "dedup_dcs_abs_band": 0.7214424273869453, + "dedup_dcs_abs_band_std": 0.0009275846990632673, + "matched_cos_abs_per_threshold": { + "0.5": 0.7253387769063314, + "0.6": 0.7298229734102885, + "0.7": 0.7306790749231974, + "0.8": 0.73060142993927, + "0.9": 0.7305633823076884 + }, + "matched_cos_abs_band": 0.730416715145111, + "matched_cos_abs_band_std": 0.0008016399906072152, + "unique_frac_abs_per_threshold": { + "0.5": 0.9609375, + "0.6": 0.9808213975694445, + "0.7": 0.9870334201388888, + "0.8": 0.9899088541666666, + "0.9": 0.9930826822916666 + }, + "unique_frac_abs_band": 0.9877115885416666, + "unique_frac_abs_band_std": 0.000317637023844364, + "matched_frac_abs_per_threshold": { + "0.5": 0.7468804253472222, + "0.6": 0.7083604600694443, + "0.7": 0.640869140625, + "0.8": 0.537109375, + "0.9": 0.36965603298611116 + }, + "matched_frac_abs_band": 0.5639987521701388, + "matched_frac_abs_band_std": 0.0018867590769537008, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 12288 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed0/config.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..10dca0d51af6f32bd6a0e03266fe81f53950f413 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed0/lock.lock b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed0/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..4afbca338ba0551880f288cbb656dcf105d3e15e --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.30206723632813, + "l0_per_token_0.001": 64.30206723632813, + "l0_per_token_0.01": 64.30206723632813, + "l0_per_token_0.05": 64.30206723632813, + "l0_per_token_0.1": 64.30206723632813, + "l0_per_token_0.5": 53.97480439453125, + "l0_per_token_1.0": 20.09258876953125, + "l1_per_token": 77.41030274099167, + "r2_score": 0.9346842268803025, + "l2_loss": 0.043485346797152435 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed1/config.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..854438ca02d0ba74a3c53bd7168fcd09f7a150fb --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed1/lock.lock b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed1/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..660a3c24705893ed44d55609cecc67b78c2bb69f --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.53487338867187, + "l0_per_token_0.001": 64.53487338867187, + "l0_per_token_0.01": 64.53487338867187, + "l0_per_token_0.05": 64.53487338867187, + "l0_per_token_0.1": 64.53487338867187, + "l0_per_token_0.5": 54.15983681640625, + "l0_per_token_1.0": 20.13566376953125, + "l1_per_token": 77.39352973653504, + "r2_score": 0.9344907847309781, + "l2_loss": 0.04361413496801491 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed2/config.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f09a7c503cce76d640a14afb82902f286199463c --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed2/lock.lock b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed2/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..4acc16a0326321f56159399445d4512773dffa78 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 63.99384736328125, + "l0_per_token_0.001": 63.99384736328125, + "l0_per_token_0.01": 63.99384736328125, + "l0_per_token_0.05": 63.99384736328125, + "l0_per_token_0.1": 63.99384736328125, + "l0_per_token_0.5": 53.74799565429687, + "l0_per_token_1.0": 19.984163818359374, + "l1_per_token": 77.04586503847759, + "r2_score": 0.9557593676449077, + "l2_loss": 0.04355602770332086 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_128/code_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..d1499f265764b2db5e01ab5267234baf9f2e5cd7 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 128.48971166666666, + "recon_r2": 0.9441782114753384, + "l1_per_token": 116.87829993859435 + }, + "seed1": { + "l0": 128.896155, + "recon_r2": 0.9438209488038847, + "l1_per_token": 117.0022341906176 + }, + "seed2": { + "l0": 128.39480666666665, + "recon_r2": 0.944132029141497, + "l1_per_token": 117.0063443909422 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6873528128176187, + "mcc_abs": 0.6874380340168081, + "act_iou": 0.3797882863087001, + "act_iou_n_pairs": 3072, + "input_r2": 0.961204051644145, + "matched_mcc_signed": { + "0.6": 0.8596104774188935, + "0.7": 0.8847935027499697, + "0.8": 0.9096381949845322, + "0.9": 0.9419448536679758 + }, + "matched_mcc_abs": { + "0.6": 0.8594538287371094, + "0.7": 0.8837653054940728, + "0.8": 0.9090860199474124, + "0.9": 0.9407026823981997 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6910568672311093, + "mcc_abs": 0.6911242527301233, + "act_iou": 0.38827579147443153, + "act_iou_n_pairs": 3072, + "input_r2": 0.9622879601453894, + "matched_mcc_signed": { + "0.6": 0.8665718928235997, + "0.7": 0.8900399613127792, + "0.8": 0.9141505632179799, + "0.9": 0.9450849132375383 + }, + "matched_mcc_abs": { + "0.6": 0.8643369876489572, + "0.7": 0.8877410629974092, + "0.8": 0.9129873911362207, + "0.9": 0.9432080777778072 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6838268116919165, + "mcc_abs": 0.6838878138261585, + "act_iou": 0.3766394518071874, + "act_iou_n_pairs": 3072, + "input_r2": 0.9611774123810984, + "matched_mcc_signed": { + "0.6": 0.8567180459358079, + "0.7": 0.881588309334675, + "0.8": 0.9082150001339284, + "0.9": 0.9446999097821421 + }, + "matched_mcc_abs": { + "0.6": 0.8539797484962931, + "0.7": 0.8808899116560499, + "0.8": 0.9078878024427905, + "0.9": 0.9436577471434807 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 128.59355777777776, + "l0_std": 0.2663185103005506, + "recon_r2_mean": 0.9440437298069067, + "recon_r2_std": 0.0001943109195860642, + "l1_per_token_mean": 116.96229284005138, + "l1_per_token_std": 0.0727690116605632, + "input_r2_mean": 0.9615564747235443, + "input_r2_std": 0.0006336249713851833, + "mcc_signed_mean": 0.6874121639135482, + "mcc_signed_std": 0.0036153931583430858, + "mcc_abs_mean": 0.6874833668576966, + "mcc_abs_std": 0.003618432437195878, + "matched_mcc_signed_per_threshold": { + "0.6": 0.860966805392767, + "0.7": 0.885473924465808, + "0.8": 0.9106679194454802, + "0.9": 0.9439098922292187 + }, + "matched_mcc_signed_band": 0.9002546353833185, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8592568549607865, + "0.7": 0.8841320933825106, + "0.8": 0.9099870711754745, + "0.9": 0.9425228357731625 + }, + "matched_mcc_abs_band": 0.8989747138229836, + "act_iou_mean": 0.381567843196773, + "act_iou_std": 0.006018821915565012, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_128/dict_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..9cdb1fc31cedce5789a3fc52d5d73ff05e83d782 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 128, + "l0_raw": 128.26168481445313, + "recon_r2": 0.9443738278938557, + "l1_per_token": 116.609667876866, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 397, + "dense_rate": 0.12923177083333334, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 34, + "0.6": 4, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 17, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 17, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.20932160317897797, + "n_dup_abs": { + "0.5": 638, + "0.6": 596, + "0.7": 582, + "0.8": 576, + "0.9": 530 + }, + "n_dup_clusters_abs": { + "0.5": 319, + "0.6": 298, + "0.7": 291, + "0.8": 288, + "0.9": 265 + }, + "n_dup_removed_abs": { + "0.5": 319, + "0.6": 298, + "0.7": 291, + "0.8": 288, + "0.9": 265 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3933434784412384, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 129, + "l0_raw": 128.58213427734376, + "recon_r2": 0.9440697197468552, + "l1_per_token": 116.67219454466817, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 407, + "dense_rate": 0.13248697916666666, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 19, + "0.6": 2, + "0.7": 2, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 9, + "0.6": 1, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 10, + "0.6": 1, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.20774340629577637, + "n_dup_abs": { + "0.5": 647, + "0.6": 614, + "0.7": 592, + "0.8": 578, + "0.9": 536 + }, + "n_dup_clusters_abs": { + "0.5": 323, + "0.6": 307, + "0.7": 296, + "0.8": 289, + "0.9": 268 + }, + "n_dup_removed_abs": { + "0.5": 324, + "0.6": 307, + "0.7": 296, + "0.8": 289, + "0.9": 268 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.39394745230674744, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 128, + "l0_raw": 128.15229565429686, + "recon_r2": 0.9623674494383497, + "l1_per_token": 116.70314136799581, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 402, + "dense_rate": 0.130859375, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 40, + "0.6": 8, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 20, + "0.6": 4, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 20, + "0.6": 4, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.2093600481748581, + "n_dup_abs": { + "0.5": 656, + "0.6": 604, + "0.7": 576, + "0.8": 566, + "0.9": 532 + }, + "n_dup_clusters_abs": { + "0.5": 328, + "0.6": 302, + "0.7": 288, + "0.8": 283, + "0.9": 266 + }, + "n_dup_removed_abs": { + "0.5": 328, + "0.6": 302, + "0.7": 288, + "0.8": 283, + "0.9": 266 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3924921452999115, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.6907104849815369, + "dcs_abs": 0.6929294466972351, + "frac_sign_flipped": 0.042317708333333336, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.6900410652160645, + "0.6": 0.6908091902732849, + "0.7": 0.6909223198890686, + "0.8": 0.6907104849815369, + "0.9": 0.6907104849815369 + }, + "unique_frac_signed": { + "0.5": 0.99560546875, + "0.6": 0.99951171875, + "0.7": 0.9998372395833334, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6870086581911892, + "0.6": 0.6904718810983468, + "0.7": 0.6908098650843992, + "0.8": 0.6907104849815369, + "0.9": 0.6907104849815369 + }, + "matched_frac_signed": { + "0.5": 0.7415364583333334, + "0.6": 0.6858723958333334, + "0.7": 0.6149088541666666, + "0.8": 0.5094401041666666, + "0.9": 0.2874348958333333 + }, + "matched_cos_abs": { + "0.5": 0.7124983668327332, + "0.6": 0.7135192155838013, + "0.7": 0.712742030620575, + "0.8": 0.7115932106971741, + "0.9": 0.7091257572174072 + }, + "unique_frac_abs": { + "0.5": 0.8953450520833334, + "0.6": 0.9015299479166666, + "0.7": 0.9044596354166666, + "0.8": 0.9060872395833334, + "0.9": 0.9132486979166666 + }, + "dedup_dcs_abs": { + "0.5": 0.6379318873611435, + "0.6": 0.6432589412628051, + "0.7": 0.6446463971612199, + "0.8": 0.6447655279868438, + "0.9": 0.6476081744379674 + }, + "matched_frac_abs": { + "0.5": 0.6956380208333334, + "0.6": 0.6500651041666666, + "0.7": 0.5875651041666666, + "0.8": 0.48828125, + "0.9": 0.2783203125 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.6939118504524231, + "dcs_abs": 0.6963305473327637, + "frac_sign_flipped": 0.0439453125, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.6926676630973816, + "0.6": 0.6940744519233704, + "0.7": 0.6939118504524231, + "0.8": 0.6939118504524231, + "0.9": 0.6939118504524231 + }, + "unique_frac_signed": { + "0.5": 0.9939778645833334, + "0.6": 0.9990234375, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6884963246314632, + "0.6": 0.693396644841414, + "0.7": 0.6939118504524231, + "0.8": 0.6939118504524231, + "0.9": 0.6939118504524231 + }, + "matched_frac_signed": { + "0.5": 0.7386067708333334, + "0.6": 0.6881510416666666, + "0.7": 0.6207682291666666, + "0.8": 0.5201822916666666, + "0.9": 0.3024088541666667 + }, + "matched_cos_abs": { + "0.5": 0.7181602120399475, + "0.6": 0.7187346816062927, + "0.7": 0.718349039554596, + "0.8": 0.7182344794273376, + "0.9": 0.7154276371002197 + }, + "unique_frac_abs": { + "0.5": 0.8946940104166666, + "0.6": 0.90234375, + "0.7": 0.90576171875, + "0.8": 0.9070638020833334, + "0.9": 0.91357421875 + }, + "dedup_dcs_abs": { + "0.5": 0.6425336402317043, + "0.6": 0.6485457478556782, + "0.7": 0.6506530607293826, + "0.8": 0.6514844976967046, + "0.9": 0.6535962446359918 + }, + "matched_frac_abs": { + "0.5": 0.6975911458333334, + "0.6": 0.6608072916666666, + "0.7": 0.6002604166666666, + "0.8": 0.5022786458333334, + "0.9": 0.2936197916666667 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.6889660954475403, + "dcs_abs": 0.6914485096931458, + "frac_sign_flipped": 0.045247395833333336, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.6888657212257385, + "0.6": 0.6892459988594055, + "0.7": 0.6891716718673706, + "0.8": 0.6889660954475403, + "0.9": 0.6889660954475403 + }, + "unique_frac_signed": { + "0.5": 0.9951171875, + "0.6": 0.9991861979166666, + "0.7": 0.9998372395833334, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.685502119071316, + "0.6": 0.6886850890296046, + "0.7": 0.6890595019989026, + "0.8": 0.6889660954475403, + "0.9": 0.6889660954475403 + }, + "matched_frac_signed": { + "0.5": 0.7425130208333334, + "0.6": 0.6822916666666666, + "0.7": 0.6097005208333334, + "0.8": 0.5016276041666666, + "0.9": 0.2737630208333333 + }, + "matched_cos_abs": { + "0.5": 0.711232602596283, + "0.6": 0.7117265462875366, + "0.7": 0.7115870714187622, + "0.8": 0.7103878259658813, + "0.9": 0.7080684900283813 + }, + "unique_frac_abs": { + "0.5": 0.8938802083333334, + "0.6": 0.90087890625, + "0.7": 0.9049479166666666, + "0.8": 0.9069010416666666, + "0.9": 0.9130859375 + }, + "dedup_dcs_abs": { + "0.5": 0.6357567469822243, + "0.6": 0.641179432568606, + "0.7": 0.6439492378073434, + "0.8": 0.6442514593557764, + "0.9": 0.646527381031774 + }, + "matched_frac_abs": { + "0.5": 0.6985677083333334, + "0.6": 0.6507161458333334, + "0.7": 0.5843098958333334, + "0.8": 0.4807942708333333, + "0.9": 0.2646484375 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 402, + "n_dense_std": 5.0, + "l0_mean": 128, + "l0_std": 0.22338870774557548, + "recon_r2_mean": 0.9502703323596869, + "recon_r2_std": 0.010477514096980523, + "l1_per_token_mean": 116.66166792984332, + "l1_per_token_std": 0.04761754509371334, + "babel_mu1_signed_mean": 0.20880835254987082, + "babel_mu1_signed_std": 0.0009224708111964264, + "n_dup_signed_per_threshold": { + "0.5": 31.0, + "0.6": 4.666666666666667, + "0.7": 0.6666666666666666, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 1.3333333333333335, + "n_dup_signed_band_std": 0.5773502691896257, + "n_dup_clusters_signed_per_threshold": { + "0.5": 15.333333333333334, + "0.6": 2.3333333333333335, + "0.7": 0.3333333333333333, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.6666666666666667, + "n_dup_clusters_signed_band_std": 0.28867513459481287, + "largest_dup_signed_per_threshold": { + "0.5": 2.3333333333333335, + "0.6": 2.0, + "0.7": 1.3333333333333333, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.3333333333333333, + "largest_dup_signed_band_std": 0.14433756729740646, + "dcs_signed_mean": 0.6911961436271667, + "dcs_signed_std": 0.002508390197345129, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.6870023672979895, + "0.6": 0.6908512049897885, + "0.7": 0.6912604058452416, + "0.8": 0.6911961436271667, + "0.9": 0.6911961436271667 + }, + "dedup_dcs_signed_band": 0.6911259745223409, + "dedup_dcs_signed_band_std": 0.0024629946006996094, + "matched_cos_signed_per_threshold": { + "0.5": 0.6905248165130615, + "0.6": 0.6913765470186869, + "0.7": 0.6913352807362875, + "0.8": 0.6911961436271667, + "0.9": 0.6911961436271667 + }, + "matched_cos_signed_band": 0.691276028752327, + "matched_cos_signed_band_std": 0.002468943820049372, + "unique_frac_signed_per_threshold": { + "0.5": 0.9949001736111112, + "0.6": 0.9992404513888888, + "0.7": 0.9998914930555557, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9997829861111112, + "unique_frac_signed_band_std": 4.698488518797919e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.7408854166666666, + "0.6": 0.6854383680555555, + "0.7": 0.6151258680555555, + "0.8": 0.5104166666666666, + "0.9": 0.2878689236111111 + }, + "matched_frac_signed_band": 0.5247124565972222, + "matched_frac_signed_band_std": 0.008020114838977724, + "babel_mu1_abs_mean": 0.39326102534929913, + "babel_mu1_abs_std": 0.0007311487573088021, + "n_dup_abs_per_threshold": { + "0.5": 647.0, + "0.6": 604.6666666666666, + "0.7": 583.3333333333334, + "0.8": 573.3333333333334, + "0.9": 532.6666666666666 + }, + "n_dup_abs_band": 573.5, + "n_dup_abs_band_std": 5.678908345800274, + "n_dup_clusters_abs_per_threshold": { + "0.5": 323.3333333333333, + "0.6": 302.3333333333333, + "0.7": 291.6666666666667, + "0.8": 286.6666666666667, + "0.9": 266.3333333333333 + }, + "n_dup_clusters_abs_band": 286.75, + "n_dup_clusters_abs_band_std": 2.839454172900137, + "largest_dup_abs_per_threshold": { + "0.5": 2.3333333333333335, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.0, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.6935695012410482, + "dcs_abs_std": 0.0025031630476166243, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6387407581916907, + "0.6": 0.6443280405623631, + "0.7": 0.6464162318993152, + "0.8": 0.6468338283464415, + "0.9": 0.6492439333685777 + }, + "dedup_dcs_abs_band": 0.6467055085441744, + "dedup_dcs_abs_band_std": 0.0038189595842243753, + "matched_cos_abs_per_threshold": { + "0.5": 0.7139637271563212, + "0.6": 0.7146601478258768, + "0.7": 0.7142260471979777, + "0.8": 0.713405172030131, + "0.9": 0.7108739614486694 + }, + "matched_cos_abs_band": 0.7132913321256638, + "matched_cos_abs_band_std": 0.0038616097916974817, + "unique_frac_abs_per_threshold": { + "0.5": 0.8946397569444445, + "0.6": 0.9015842013888888, + "0.7": 0.905056423611111, + "0.8": 0.9066840277777778, + "0.9": 0.9133029513888888 + }, + "unique_frac_abs_band": 0.9066569010416666, + "unique_frac_abs_band_std": 0.0004621507442871952, + "matched_frac_abs_per_threshold": { + "0.5": 0.697265625, + "0.6": 0.6538628472222222, + "0.7": 0.5907118055555555, + "0.8": 0.4904513888888889, + "0.9": 0.27886284722222227 + }, + "matched_frac_abs_band": 0.5034722222222222, + "matched_frac_abs_band_std": 0.009788091741928729, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 3072 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed0/config.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c35a8861a4210a98ca2e367ff6d4aa8deaefeb55 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..87e7f2980017dd9f0cbfaf34dd3438daeb56223e --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.26168481445313, + "l0_per_token_0.001": 128.26168481445313, + "l0_per_token_0.01": 128.26168481445313, + "l0_per_token_0.05": 128.26168481445313, + "l0_per_token_0.1": 128.26168481445313, + "l0_per_token_0.5": 84.78793833007812, + "l0_per_token_1.0": 26.861700341796876, + "l1_per_token": 116.609667876866, + "r2_score": 0.9443738278938557, + "l2_loss": 0.037034291557735376 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed1/config.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..69eb8061c95e13eabf11b5605ce9e0273fa5b6ee --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..872602570cc14f1c411e93cc2ad4b53ea026fe4d --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.58213427734376, + "l0_per_token_0.001": 128.58213427734376, + "l0_per_token_0.01": 128.58213427734376, + "l0_per_token_0.05": 128.58213427734376, + "l0_per_token_0.1": 128.58213427734376, + "l0_per_token_0.5": 84.93437954101563, + "l0_per_token_1.0": 26.862887158203126, + "l1_per_token": 116.67219454466817, + "r2_score": 0.9440697197468552, + "l2_loss": 0.037236757939200685 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed2/config.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ea55bca5defbb5c0d14766fcc72e07889904e940 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e072d46d28ecebf8b7e8dd8a900a133a1b634b2d --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.15229565429686, + "l0_per_token_0.001": 128.15229565429686, + "l0_per_token_0.01": 128.15229565429686, + "l0_per_token_0.05": 128.15229565429686, + "l0_per_token_0.1": 128.15229565429686, + "l0_per_token_0.5": 84.69056245117187, + "l0_per_token_1.0": 26.839740576171874, + "l1_per_token": 116.70314136799581, + "r2_score": 0.9623674494383497, + "l2_loss": 0.03705020311765927 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_32/code_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..6446e3b2b02bb2a83ed233fa3cec2db463b8a648 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 32.06041166666667, + "recon_r2": 0.8959020009198049, + "l1_per_token": 59.40846914071481 + }, + "seed1": { + "l0": 32.269643333333335, + "recon_r2": 0.8957378962830598, + "l1_per_token": 60.11258848291228 + }, + "seed2": { + "l0": 32.12463, + "recon_r2": 0.8959581012482468, + "l1_per_token": 59.77764700556427 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7777810708128156, + "mcc_abs": 0.7778094784156583, + "act_iou": 0.4970233528222276, + "act_iou_n_pairs": 3072, + "input_r2": 0.9682769536661923, + "matched_mcc_signed": { + "0.6": 0.8695836159242532, + "0.7": 0.8888210600128459, + "0.8": 0.9083680960348217, + "0.9": 0.9317697597599308 + }, + "matched_mcc_abs": { + "0.6": 0.8678783390735958, + "0.7": 0.8870357266672131, + "0.8": 0.9077634770466967, + "0.9": 0.931861201167147 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7827728803654042, + "mcc_abs": 0.7827991449628943, + "act_iou": 0.5048010112485251, + "act_iou_n_pairs": 3072, + "input_r2": 0.9687737664642649, + "matched_mcc_signed": { + "0.6": 0.8727913160977104, + "0.7": 0.8920775979234887, + "0.8": 0.9096951034478316, + "0.9": 0.9330585255803823 + }, + "matched_mcc_abs": { + "0.6": 0.8711784105519209, + "0.7": 0.889917715030348, + "0.8": 0.9080923918775562, + "0.9": 0.9324462686409495 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7798329076671457, + "mcc_abs": 0.7798604145324743, + "act_iou": 0.5008110767106883, + "act_iou_n_pairs": 3072, + "input_r2": 0.9679605798661924, + "matched_mcc_signed": { + "0.6": 0.8714800911975222, + "0.7": 0.8887704033644741, + "0.8": 0.906546568035108, + "0.9": 0.9310157923746245 + }, + "matched_mcc_abs": { + "0.6": 0.8705399761966911, + "0.7": 0.8883702690203916, + "0.8": 0.9063357967550932, + "0.9": 0.930426839359609 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 32.15156166666667, + "l0_std": 0.10718422731965342, + "recon_r2_mean": 0.8958659994837038, + "recon_r2_std": 0.00011443178852163545, + "l1_per_token_mean": 59.766234876397114, + "l1_per_token_std": 0.3521983667384705, + "input_r2_mean": 0.9683370999988833, + "input_r2_std": 0.0004099162070756778, + "mcc_signed_mean": 0.7801289529484552, + "mcc_signed_std": 0.0025090382138915047, + "mcc_abs_mean": 0.7801563459703423, + "mcc_abs_std": 0.002507962245547251, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8712850077398286, + "0.7": 0.8898896871002696, + "0.8": 0.9082032558392538, + "0.9": 0.9319480259049792 + }, + "matched_mcc_signed_band": 0.9003314941460828, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8698655752740693, + "0.7": 0.8884412369059843, + "0.8": 0.9073972218931154, + "0.9": 0.9315781030559019 + }, + "matched_mcc_abs_band": 0.8993205342822678, + "act_iou_mean": 0.5008784802604803, + "act_iou_std": 0.0038892672931970626, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_32/dict_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..068ffac08c2e374080a52fb81bc722f862149866 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 32, + "l0_raw": 31.959387255859376, + "recon_r2": 0.8962818178087888, + "l1_per_token": 59.255121427897826, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 21, + "dense_rate": 0.0068359375, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 186, + "0.6": 44, + "0.7": 10, + "0.8": 3, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 91, + "0.6": 21, + "0.7": 4, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 95, + "0.6": 23, + "0.7": 6, + "0.8": 2, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 4, + "0.6": 4, + "0.7": 4, + "0.8": 3, + "0.9": 1 + }, + "babel_mu1_signed": 0.3193931579589844, + "n_dup_abs": { + "0.5": 262, + "0.6": 104, + "0.7": 60, + "0.8": 45, + "0.9": 24 + }, + "n_dup_clusters_abs": { + "0.5": 129, + "0.6": 51, + "0.7": 29, + "0.8": 22, + "0.9": 12 + }, + "n_dup_removed_abs": { + "0.5": 133, + "0.6": 53, + "0.7": 31, + "0.8": 23, + "0.9": 12 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 4, + "0.7": 4, + "0.8": 3, + "0.9": 2 + }, + "babel_mu1_abs": 0.33999451994895935, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 32, + "l0_raw": 32.14736723632812, + "recon_r2": 0.8962231109551728, + "l1_per_token": 59.92669258867342, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 22, + "dense_rate": 0.007161458333333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 185, + "0.6": 39, + "0.7": 11, + "0.8": 4, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 88, + "0.6": 18, + "0.7": 4, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 97, + "0.6": 21, + "0.7": 7, + "0.8": 3, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 5, + "0.7": 5, + "0.8": 4, + "0.9": 1 + }, + "babel_mu1_signed": 0.31915634870529175, + "n_dup_abs": { + "0.5": 247, + "0.6": 91, + "0.7": 59, + "0.8": 44, + "0.9": 22 + }, + "n_dup_clusters_abs": { + "0.5": 119, + "0.6": 44, + "0.7": 28, + "0.8": 21, + "0.9": 11 + }, + "n_dup_removed_abs": { + "0.5": 128, + "0.6": 47, + "0.7": 31, + "0.8": 23, + "0.9": 11 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 5, + "0.7": 5, + "0.8": 4, + "0.9": 2 + }, + "babel_mu1_abs": 0.3383723795413971, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 32, + "l0_raw": 32.02102919921875, + "recon_r2": 0.9299208191231755, + "l1_per_token": 59.61213497639423, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 22, + "dense_rate": 0.007161458333333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 186, + "0.6": 32, + "0.7": 13, + "0.8": 4, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 90, + "0.6": 14, + "0.7": 5, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 96, + "0.6": 18, + "0.7": 8, + "0.8": 3, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 5, + "0.7": 5, + "0.8": 4, + "0.9": 1 + }, + "babel_mu1_signed": 0.31979504227638245, + "n_dup_abs": { + "0.5": 258, + "0.6": 90, + "0.7": 63, + "0.8": 46, + "0.9": 30 + }, + "n_dup_clusters_abs": { + "0.5": 126, + "0.6": 43, + "0.7": 30, + "0.8": 22, + "0.9": 15 + }, + "n_dup_removed_abs": { + "0.5": 132, + "0.6": 47, + "0.7": 33, + "0.8": 24, + "0.9": 15 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 5, + "0.7": 5, + "0.8": 4, + "0.9": 2 + }, + "babel_mu1_abs": 0.3402721881866455, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8309513926506042, + "dcs_abs": 0.8323795795440674, + "frac_sign_flipped": 0.015625, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8262704610824585, + "0.6": 0.8298234343528748, + "0.7": 0.8306627869606018, + "0.8": 0.8310738801956177, + "0.9": 0.8309513926506042 + }, + "unique_frac_signed": { + "0.5": 0.96875, + "0.6": 0.9928385416666666, + "0.7": 0.9978841145833334, + "0.8": 0.9991861979166666, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.8004495091736317, + "0.6": 0.8238806884037331, + "0.7": 0.8289051996835042, + "0.8": 0.8303975505405106, + "0.9": 0.8309513926506042 + }, + "matched_frac_signed": { + "0.5": 0.8541666666666666, + "0.6": 0.845703125, + "0.7": 0.8046875, + "0.8": 0.736328125, + "0.9": 0.6077473958333334 + }, + "matched_cos_abs": { + "0.5": 0.8279819488525391, + "0.6": 0.8319348692893982, + "0.7": 0.8319737911224365, + "0.8": 0.8325097560882568, + "0.9": 0.8322185277938843 + }, + "unique_frac_abs": { + "0.5": 0.95751953125, + "0.6": 0.9837239583333334, + "0.7": 0.9899088541666666, + "0.8": 0.9925130208333334, + "0.9": 0.9962565104166666 + }, + "dedup_dcs_abs": { + "0.5": 0.7928088875487447, + "0.6": 0.8183942626928911, + "0.7": 0.8235782222667087, + "0.8": 0.8262767728883773, + "0.9": 0.8291031264040308 + }, + "matched_frac_abs": { + "0.5": 0.845703125, + "0.6": 0.8401692708333334, + "0.7": 0.8020833333333334, + "0.8": 0.7333984375, + "0.9": 0.6048177083333334 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8359336256980896, + "dcs_abs": 0.8371616005897522, + "frac_sign_flipped": 0.013671875, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8306071162223816, + "0.6": 0.8346603512763977, + "0.7": 0.8358731269836426, + "0.8": 0.8360882997512817, + "0.9": 0.8359336256980896 + }, + "unique_frac_signed": { + "0.5": 0.9689127604166666, + "0.6": 0.9933268229166666, + "0.7": 0.9977213541666666, + "0.8": 0.9991861979166666, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.8047858338007547, + "0.6": 0.8290905149478931, + "0.7": 0.833968468165646, + "0.8": 0.8354078893510934, + "0.9": 0.8359336256980896 + }, + "matched_frac_signed": { + "0.5": 0.8528645833333334, + "0.6": 0.8509114583333334, + "0.7": 0.8121744791666666, + "0.8": 0.751953125, + "0.9": 0.6171875 + }, + "matched_cos_abs": { + "0.5": 0.832356870174408, + "0.6": 0.83656907081604, + "0.7": 0.8370628952980042, + "0.8": 0.8374587297439575, + "0.9": 0.8373229503631592 + }, + "unique_frac_abs": { + "0.5": 0.9568684895833334, + "0.6": 0.9837239583333334, + "0.7": 0.9895833333333334, + "0.8": 0.9923502604166666, + "0.9": 0.99560546875 + }, + "dedup_dcs_abs": { + "0.5": 0.7964560611580964, + "0.6": 0.8229530377623936, + "0.7": 0.8283434901386499, + "0.8": 0.831052388549627, + "0.9": 0.8336433084914461 + }, + "matched_frac_abs": { + "0.5": 0.84375, + "0.6": 0.845703125, + "0.7": 0.8082682291666666, + "0.8": 0.7496744791666666, + "0.9": 0.6149088541666666 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.833625316619873, + "dcs_abs": 0.8348724246025085, + "frac_sign_flipped": 0.0146484375, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8281179666519165, + "0.6": 0.8330881595611572, + "0.7": 0.833500325679779, + "0.8": 0.8334960341453552, + "0.9": 0.8336253762245178 + }, + "unique_frac_signed": { + "0.5": 0.9685872395833334, + "0.6": 0.99365234375, + "0.7": 0.99755859375, + "0.8": 0.9990234375, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.8021044953687427, + "0.6": 0.8278000022983178, + "0.7": 0.8314654127752874, + "0.8": 0.8326820731745102, + "0.9": 0.8336253762245178 + }, + "matched_frac_signed": { + "0.5": 0.8512369791666666, + "0.6": 0.8450520833333334, + "0.7": 0.810546875, + "0.8": 0.7503255208333334, + "0.9": 0.61328125 + }, + "matched_cos_abs": { + "0.5": 0.8291143774986267, + "0.6": 0.8335397839546204, + "0.7": 0.8345054388046265, + "0.8": 0.8345841765403748, + "0.9": 0.8348575830459595 + }, + "unique_frac_abs": { + "0.5": 0.9576822916666666, + "0.6": 0.9847005208333334, + "0.7": 0.9895833333333334, + "0.8": 0.9923502604166666, + "0.9": 0.9957682291666666 + }, + "dedup_dcs_abs": { + "0.5": 0.7940281570966665, + "0.6": 0.8207870593954188, + "0.7": 0.8258126738170783, + "0.8": 0.8281998249294702, + "0.9": 0.8313246570760384 + }, + "matched_frac_abs": { + "0.5": 0.8424479166666666, + "0.6": 0.8391927083333334, + "0.7": 0.8043619791666666, + "0.8": 0.7464192708333334, + "0.9": 0.6106770833333334 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 22, + "n_dense_std": 0.5773502691896258, + "l0_mean": 32, + "l0_std": 0.09582753767298095, + "recon_r2_mean": 0.9074752492957123, + "recon_r2_std": 0.019438455835876143, + "l1_per_token_mean": 59.59798299765516, + "l1_per_token_std": 0.336009173792982, + "babel_mu1_signed_mean": 0.31944818298021954, + "babel_mu1_signed_std": 0.00032288261668847023, + "n_dup_signed_per_threshold": { + "0.5": 185.66666666666666, + "0.6": 38.333333333333336, + "0.7": 11.333333333333334, + "0.8": 3.6666666666666665, + "0.9": 0.0 + }, + "n_dup_signed_band": 13.333333333333334, + "n_dup_signed_band_std": 1.0103629710818451, + "n_dup_clusters_signed_per_threshold": { + "0.5": 89.66666666666667, + "0.6": 17.666666666666668, + "0.7": 4.333333333333333, + "0.8": 1.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 5.75, + "n_dup_clusters_signed_band_std": 0.75, + "largest_dup_signed_per_threshold": { + "0.5": 5.333333333333333, + "0.6": 4.666666666666667, + "0.7": 4.666666666666667, + "0.8": 3.6666666666666665, + "0.9": 1.0 + }, + "largest_dup_signed_band": 3.5, + "largest_dup_signed_band_std": 0.4330127018922193, + "dcs_signed_mean": 0.8335034449895223, + "dcs_signed_std": 0.002493351370259927, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.802446612781043, + "0.6": 0.826923735216648, + "0.7": 0.8314463602081458, + "0.8": 0.832829171022038, + "0.9": 0.8335034648577372 + }, + "dedup_dcs_signed_band": 0.8311756828261422, + "dedup_dcs_signed_band_std": 0.0025402037610776436, + "matched_cos_signed_per_threshold": { + "0.5": 0.8283318479855856, + "0.6": 0.8325239817301432, + "0.7": 0.8333454132080078, + "0.8": 0.8335527380307516, + "0.9": 0.8335034648577372 + }, + "matched_cos_signed_band": 0.8332313994566599, + "matched_cos_signed_band_std": 0.0025112362464775224, + "unique_frac_signed_per_threshold": { + "0.5": 0.96875, + "0.6": 0.9932725694444443, + "0.7": 0.9977213541666666, + "0.8": 0.9991319444444443, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9975314670138888, + "unique_frac_signed_band_std": 4.698488518797919e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.8527560763888888, + "0.6": 0.8472222222222223, + "0.7": 0.8091362847222222, + "0.8": 0.7462022569444445, + "0.9": 0.6127387152777778 + }, + "matched_frac_signed_band": 0.7538248697916667, + "matched_frac_signed_band_std": 0.004795221309458398, + "babel_mu1_abs_mean": 0.33954636255900067, + "babel_mu1_abs_std": 0.0010261344974081604, + "n_dup_abs_per_threshold": { + "0.5": 255.66666666666666, + "0.6": 95.0, + "0.7": 60.666666666666664, + "0.8": 45.0, + "0.9": 25.333333333333332 + }, + "n_dup_abs_band": 56.5, + "n_dup_abs_band_std": 2.222048604328897, + "n_dup_clusters_abs_per_threshold": { + "0.5": 124.66666666666667, + "0.6": 46.0, + "0.7": 29.0, + "0.8": 21.666666666666668, + "0.9": 12.666666666666666 + }, + "n_dup_clusters_abs_band": 27.333333333333336, + "n_dup_clusters_abs_band_std": 1.2583057392117916, + "largest_dup_abs_per_threshold": { + "0.5": 5.333333333333333, + "0.6": 4.666666666666667, + "0.7": 4.666666666666667, + "0.8": 3.6666666666666665, + "0.9": 2.0 + }, + "largest_dup_abs_band": 3.75, + "largest_dup_abs_band_std": 0.4330127018922193, + "dcs_abs_mean": 0.8348045349121094, + "dcs_abs_std": 0.002391733279002906, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.7944310352678358, + "0.6": 0.8207114532835679, + "0.7": 0.8259114620741457, + "0.8": 0.8285096621224914, + "0.9": 0.8313570306571717 + }, + "dedup_dcs_abs_band": 0.8266224020343441, + "dedup_dcs_abs_band_std": 0.0023313227096879623, + "matched_cos_abs_per_threshold": { + "0.5": 0.8298177321751913, + "0.6": 0.8340145746866862, + "0.7": 0.8345140417416891, + "0.8": 0.8348508874575297, + "0.9": 0.8347996870676676 + }, + "matched_cos_abs_band": 0.8345447977383931, + "matched_cos_abs_band_std": 0.0024766263513461523, + "unique_frac_abs_per_threshold": { + "0.5": 0.9573567708333334, + "0.6": 0.9840494791666666, + "0.7": 0.9896918402777778, + "0.8": 0.9924045138888888, + "0.9": 0.995876736111111 + }, + "unique_frac_abs_band": 0.990505642361111, + "unique_frac_abs_band_std": 0.00016444709815783103, + "matched_frac_abs_per_threshold": { + "0.5": 0.8439670138888888, + "0.6": 0.8416883680555557, + "0.7": 0.8049045138888888, + "0.8": 0.7431640625, + "0.9": 0.6101345486111112 + }, + "matched_frac_abs_band": 0.749972873263889, + "matched_frac_abs_band_std": 0.0047635815332514025, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 3072 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed0/config.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6b8b0bf9a79be3eaf8cb64f0d93fcb3200327e0f --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..6a18c55acb5967f31efccbc657e5b1e73fc27ff4 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.959387255859376, + "l0_per_token_0.001": 31.959387255859376, + "l0_per_token_0.01": 31.959387255859376, + "l0_per_token_0.05": 31.959387255859376, + "l0_per_token_0.1": 31.959387255859376, + "l0_per_token_0.5": 31.959387255859376, + "l0_per_token_1.0": 19.5565123046875, + "l1_per_token": 59.255121427897826, + "r2_score": 0.8962818178087888, + "l2_loss": 0.06905255662349184 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed1/config.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..78eb67885d6081b1eee04627d0a79573862692b1 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..daa1219976e21f22aa46ed13b52c58fd1c318080 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.14736723632812, + "l0_per_token_0.001": 32.14736723632812, + "l0_per_token_0.01": 32.14736723632812, + "l0_per_token_0.05": 32.14736723632812, + "l0_per_token_0.1": 32.14736723632812, + "l0_per_token_0.5": 32.14736723632812, + "l0_per_token_1.0": 19.55947685546875, + "l1_per_token": 59.92669258867342, + "r2_score": 0.8962231109551728, + "l2_loss": 0.06909164194342182 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed2/config.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..33db7efae94f3b46add2453de64aeae70e044c21 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..f740c58d0c512ef6f0bdea690664d2f9b1b3a28a --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.02102919921875, + "l0_per_token_0.001": 32.02102919921875, + "l0_per_token_0.01": 32.02102919921875, + "l0_per_token_0.05": 32.02102919921875, + "l0_per_token_0.1": 32.02102919921875, + "l0_per_token_0.5": 32.02102919921875, + "l0_per_token_1.0": 19.516508544921876, + "l1_per_token": 59.61213497639423, + "r2_score": 0.9299208191231755, + "l2_loss": 0.06899473586176381 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_64/code_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..ac88d7d3fd5b64374c89fe42176eacbcc9af2b3a --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 64.31472166666667, + "recon_r2": 0.9196806862352669, + "l1_per_token": 81.0035273319302 + }, + "seed1": { + "l0": 64.68409, + "recon_r2": 0.9193690147444183, + "l1_per_token": 81.30740190876489 + }, + "seed2": { + "l0": 64.27220666666666, + "recon_r2": 0.9196071594122828, + "l1_per_token": 80.77112173261592 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7481143688552404, + "mcc_abs": 0.7481390621918167, + "act_iou": 0.44963499310656535, + "act_iou_n_pairs": 3072, + "input_r2": 0.9653264929662302, + "matched_mcc_signed": { + "0.6": 0.869842541254191, + "0.7": 0.89098746312633, + "0.8": 0.9108527952427168, + "0.9": 0.9375759435734091 + }, + "matched_mcc_abs": { + "0.6": 0.8697063041071134, + "0.7": 0.8899077153383491, + "0.8": 0.9102708809895519, + "0.9": 0.9368580808662496 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7497889963167985, + "mcc_abs": 0.7498202107834403, + "act_iou": 0.4529106575737088, + "act_iou_n_pairs": 3072, + "input_r2": 0.9661450546125551, + "matched_mcc_signed": { + "0.6": 0.8714814400292902, + "0.7": 0.8902371729913664, + "0.8": 0.9116209715282855, + "0.9": 0.9381684762726012 + }, + "matched_mcc_abs": { + "0.6": 0.8712434943275021, + "0.7": 0.8904248558493616, + "0.8": 0.9112847213474626, + "0.9": 0.9376005058118028 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7478525603196564, + "mcc_abs": 0.7478972685915064, + "act_iou": 0.4502170085274478, + "act_iou_n_pairs": 3072, + "input_r2": 0.9657202826255005, + "matched_mcc_signed": { + "0.6": 0.8702828387446647, + "0.7": 0.8911023049402776, + "0.8": 0.9115883658485436, + "0.9": 0.9385277198748214 + }, + "matched_mcc_abs": { + "0.6": 0.8693394693560088, + "0.7": 0.8907825910984616, + "0.8": 0.9118021281395805, + "0.9": 0.9384011232486195 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 64.42367277777777, + "l0_std": 0.22652754474611547, + "recon_r2_mean": 0.9195522867973226, + "recon_r2_std": 0.00016292032548567413, + "l1_per_token_mean": 81.02735032443701, + "l1_per_token_std": 0.2689326273652478, + "input_r2_mean": 0.9657306100680952, + "input_r2_std": 0.0004093785341971839, + "mcc_signed_mean": 0.7485853084972317, + "mcc_signed_std": 0.001050611347052827, + "mcc_abs_mean": 0.7486188471889211, + "mcc_abs_std": 0.0010474120015374865, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8705356066760487, + "0.7": 0.8907756470193248, + "0.8": 0.9113540442065152, + "0.9": 0.9380907132402774 + }, + "matched_mcc_signed_band": 0.9026890027855415, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8700964225968747, + "0.7": 0.8903717207620575, + "0.8": 0.9111192434921983, + "0.9": 0.9376199033088906 + }, + "matched_mcc_abs_band": 0.9023018225400052, + "act_iou_mean": 0.450920886402574, + "act_iou_std": 0.001747591906780775, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_64/dict_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..ccdcc72a694b9421fc3403006207bfba885dd083 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 64, + "l0_raw": 64.12594956054687, + "recon_r2": 0.9199404659849154, + "l1_per_token": 80.75087763596787, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 93, + "dense_rate": 0.0302734375, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 60, + "0.6": 12, + "0.7": 4, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 30, + "0.6": 6, + "0.7": 2, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 30, + "0.6": 6, + "0.7": 2, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.24736708402633667, + "n_dup_abs": { + "0.5": 258, + "0.6": 192, + "0.7": 172, + "0.8": 148, + "0.9": 132 + }, + "n_dup_clusters_abs": { + "0.5": 129, + "0.6": 96, + "0.7": 86, + "0.8": 74, + "0.9": 66 + }, + "n_dup_removed_abs": { + "0.5": 129, + "0.6": 96, + "0.7": 86, + "0.8": 74, + "0.9": 66 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3116808831691742, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 64, + "l0_raw": 64.47314228515626, + "recon_r2": 0.9197213487439466, + "l1_per_token": 81.03433132431691, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 93, + "dense_rate": 0.0302734375, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 56, + "0.6": 7, + "0.7": 5, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 27, + "0.6": 3, + "0.7": 2, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 29, + "0.6": 4, + "0.7": 3, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.24859662353992462, + "n_dup_abs": { + "0.5": 252, + "0.6": 185, + "0.7": 165, + "0.8": 152, + "0.9": 116 + }, + "n_dup_clusters_abs": { + "0.5": 125, + "0.6": 92, + "0.7": 82, + "0.8": 76, + "0.9": 58 + }, + "n_dup_removed_abs": { + "0.5": 127, + "0.6": 93, + "0.7": 83, + "0.8": 76, + "0.9": 58 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.31131118535995483, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 64, + "l0_raw": 64.09163022460937, + "recon_r2": 0.9458493843498781, + "l1_per_token": 80.53026138534722, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 91, + "dense_rate": 0.029622395833333332, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 51, + "0.6": 8, + "0.7": 4, + "0.8": 3, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 24, + "0.6": 3, + "0.7": 1, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 27, + "0.6": 5, + "0.7": 3, + "0.8": 2, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 5, + "0.6": 4, + "0.7": 4, + "0.8": 3, + "0.9": 1 + }, + "babel_mu1_signed": 0.2504028379917145, + "n_dup_abs": { + "0.5": 253, + "0.6": 188, + "0.7": 168, + "0.8": 153, + "0.9": 130 + }, + "n_dup_clusters_abs": { + "0.5": 125, + "0.6": 93, + "0.7": 83, + "0.8": 76, + "0.9": 65 + }, + "n_dup_removed_abs": { + "0.5": 128, + "0.6": 95, + "0.7": 85, + "0.8": 77, + "0.9": 65 + }, + "largest_dup_abs": { + "0.5": 5, + "0.6": 4, + "0.7": 4, + "0.8": 3, + "0.9": 2 + }, + "babel_mu1_abs": 0.31384536623954773, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.7826647162437439, + "dcs_abs": 0.7839650511741638, + "frac_sign_flipped": 0.020833333333333332, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.780045747756958, + "0.6": 0.7828252911567688, + "0.7": 0.7828103303909302, + "0.8": 0.7826198935508728, + "0.9": 0.7826647162437439 + }, + "unique_frac_signed": { + "0.5": 0.9903971354166666, + "0.6": 0.9983723958333334, + "0.7": 0.9991861979166666, + "0.8": 0.9996744791666666, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.772555074072443, + "0.6": 0.78155116145111, + "0.7": 0.7821732777132031, + "0.8": 0.7823651344709408, + "0.9": 0.7826647162437439 + }, + "matched_frac_signed": { + "0.5": 0.8229166666666666, + "0.6": 0.7919921875, + "0.7": 0.7376302083333334, + "0.8": 0.6604817708333334, + "0.9": 0.5013020833333334 + }, + "matched_cos_abs": { + "0.5": 0.7860082387924194, + "0.6": 0.7878342270851135, + "0.7": 0.7876414656639099, + "0.8": 0.7867862582206726, + "0.9": 0.787226140499115 + }, + "unique_frac_abs": { + "0.5": 0.9583333333333334, + "0.6": 0.96923828125, + "0.7": 0.9724934895833334, + "0.8": 0.9755859375, + "0.9": 0.9798177083333334 + }, + "dedup_dcs_abs": { + "0.5": 0.753257895509402, + "0.6": 0.7635990921698976, + "0.7": 0.765976197484027, + "0.8": 0.767577609338332, + "0.9": 0.7713381129239376 + }, + "matched_frac_abs": { + "0.5": 0.802734375, + "0.6": 0.7744140625, + "0.7": 0.7259114583333334, + "0.8": 0.6520182291666666, + "0.9": 0.4964192708333333 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.7842113375663757, + "dcs_abs": 0.7857304215431213, + "frac_sign_flipped": 0.021484375, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7821143865585327, + "0.6": 0.7842446565628052, + "0.7": 0.7844415903091431, + "0.8": 0.7844199538230896, + "0.9": 0.7842112183570862 + }, + "unique_frac_signed": { + "0.5": 0.99072265625, + "0.6": 0.9982096354166666, + "0.7": 0.9991861979166666, + "0.8": 0.99951171875, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7748584425426088, + "0.6": 0.7828405727050267, + "0.7": 0.7838032101086961, + "0.8": 0.7840369362675119, + "0.9": 0.7842112183570862 + }, + "matched_frac_signed": { + "0.5": 0.8291015625, + "0.6": 0.7900390625, + "0.7": 0.7392578125, + "0.8": 0.6604817708333334, + "0.9": 0.5009765625 + }, + "matched_cos_abs": { + "0.5": 0.7872470617294312, + "0.6": 0.789455235004425, + "0.7": 0.7893394231796265, + "0.8": 0.7888481020927429, + "0.9": 0.7880243062973022 + }, + "unique_frac_abs": { + "0.5": 0.9581705729166666, + "0.6": 0.9689127604166666, + "0.7": 0.97216796875, + "0.8": 0.9754231770833334, + "0.9": 0.9786783854166666 + }, + "dedup_dcs_abs": { + "0.5": 0.7543169681642514, + "0.6": 0.7649132509735257, + "0.7": 0.7673705036868341, + "0.8": 0.769460721979461, + "0.9": 0.7712223557561325 + }, + "matched_frac_abs": { + "0.5": 0.8053385416666666, + "0.6": 0.7731119791666666, + "0.7": 0.7268880208333334, + "0.8": 0.65234375, + "0.9": 0.4957682291666667 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.7816047668457031, + "dcs_abs": 0.7834148406982422, + "frac_sign_flipped": 0.026692708333333332, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.780350923538208, + "0.6": 0.7818396687507629, + "0.7": 0.781671404838562, + "0.8": 0.7817879319190979, + "0.9": 0.7816047668457031 + }, + "unique_frac_signed": { + "0.5": 0.9908854166666666, + "0.6": 0.99853515625, + "0.7": 0.9990234375, + "0.8": 0.99951171875, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7732383500163753, + "0.6": 0.7806943957984913, + "0.7": 0.7809080538572744, + "0.8": 0.7814061995304655, + "0.9": 0.7816047668457031 + }, + "matched_frac_signed": { + "0.5": 0.8287760416666666, + "0.6": 0.791015625, + "0.7": 0.7330729166666666, + "0.8": 0.6569010416666666, + "0.9": 0.4993489583333333 + }, + "matched_cos_abs": { + "0.5": 0.7860243320465088, + "0.6": 0.7869827747344971, + "0.7": 0.7873092889785767, + "0.8": 0.7864125967025757, + "0.9": 0.7863751649856567 + }, + "unique_frac_abs": { + "0.5": 0.95849609375, + "0.6": 0.9694010416666666, + "0.7": 0.97265625, + "0.8": 0.97509765625, + "0.9": 0.97998046875 + }, + "dedup_dcs_abs": { + "0.5": 0.7534012518590316, + "0.6": 0.7629019216013451, + "0.7": 0.7657813006080687, + "0.8": 0.766829079890158, + "0.9": 0.7706323027960025 + }, + "matched_frac_abs": { + "0.5": 0.8072916666666666, + "0.6": 0.7747395833333334, + "0.7": 0.7229817708333334, + "0.8": 0.6477864583333334, + "0.9": 0.4944661458333333 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 92, + "n_dense_std": 1.1547005383792517, + "l0_mean": 64, + "l0_std": 0.21105767662877017, + "recon_r2_mean": 0.9285037330262468, + "recon_r2_std": 0.015022174209217145, + "l1_per_token_mean": 80.771823448544, + "l1_per_token_std": 0.25268690338358096, + "babel_mu1_signed_mean": 0.24878884851932526, + "babel_mu1_signed_std": 0.001526978508888043, + "n_dup_signed_per_threshold": { + "0.5": 55.666666666666664, + "0.6": 9.0, + "0.7": 4.333333333333333, + "0.8": 2.3333333333333335, + "0.9": 0.0 + }, + "n_dup_signed_band": 3.9166666666666665, + "n_dup_signed_band_std": 0.5204164998665332, + "n_dup_clusters_signed_per_threshold": { + "0.5": 27.0, + "0.6": 4.0, + "0.7": 1.6666666666666667, + "0.8": 1.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 1.6666666666666667, + "n_dup_clusters_signed_band_std": 0.5204164998665332, + "largest_dup_signed_per_threshold": { + "0.5": 3.3333333333333335, + "0.6": 3.0, + "0.7": 3.0, + "0.8": 2.3333333333333335, + "0.9": 1.0 + }, + "largest_dup_signed_band": 2.3333333333333335, + "largest_dup_signed_band_std": 0.6291528696058959, + "dcs_signed_mean": 0.7828269402186075, + "dcs_signed_std": 0.0013108356853482285, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7735506222104757, + "0.6": 0.7816953766515428, + "0.7": 0.7822948472263912, + "0.8": 0.782602756756306, + "0.9": 0.7828269004821777 + }, + "dedup_dcs_signed_band": 0.7823549702791044, + "dedup_dcs_signed_band_std": 0.001292871304183153, + "matched_cos_signed_per_threshold": { + "0.5": 0.7808370192845663, + "0.6": 0.7829698721567789, + "0.7": 0.7829744418462118, + "0.8": 0.7829425930976868, + "0.9": 0.7828269004821777 + }, + "matched_cos_signed_band": 0.7829284518957138, + "matched_cos_signed_band_std": 0.0013129959021461917, + "unique_frac_signed_per_threshold": { + "0.5": 0.9906684027777777, + "0.6": 0.9983723958333334, + "0.7": 0.9991319444444443, + "0.8": 0.9995659722222222, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.999267578125, + "unique_frac_signed_band_std": 4.069010416668517e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.826931423611111, + "0.6": 0.791015625, + "0.7": 0.7366536458333334, + "0.8": 0.6592881944444445, + "0.9": 0.5005425347222222 + }, + "matched_frac_signed_band": 0.6718750000000001, + "matched_frac_signed_band_std": 0.0015526354189729593, + "babel_mu1_abs_mean": 0.31227914492289227, + "babel_mu1_abs_std": 0.001368925135679417, + "n_dup_abs_per_threshold": { + "0.5": 254.33333333333334, + "0.6": 188.33333333333334, + "0.7": 168.33333333333334, + "0.8": 151.0, + "0.9": 126.0 + }, + "n_dup_abs_band": 158.41666666666669, + "n_dup_abs_band_std": 3.449033681095813, + "n_dup_clusters_abs_per_threshold": { + "0.5": 126.33333333333333, + "0.6": 93.66666666666667, + "0.7": 83.66666666666667, + "0.8": 75.33333333333333, + "0.9": 63.0 + }, + "n_dup_clusters_abs_band": 78.91666666666667, + "n_dup_clusters_abs_band_std": 1.773649721149397, + "largest_dup_abs_per_threshold": { + "0.5": 3.3333333333333335, + "0.6": 3.0, + "0.7": 3.0, + "0.8": 2.3333333333333335, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.5833333333333335, + "largest_dup_abs_band_std": 0.6291528696058959, + "dcs_abs_mean": 0.7843701044718424, + "dcs_abs_std": 0.0012097643542289794, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.7536587051775617, + "0.6": 0.7638047549149228, + "0.7": 0.7663760005929766, + "0.8": 0.7679558037359837, + "0.9": 0.7710642571586909 + }, + "dedup_dcs_abs_band": 0.7673002041006435, + "dedup_dcs_abs_band_std": 0.0008665147078741539, + "matched_cos_abs_per_threshold": { + "0.5": 0.7864265441894531, + "0.6": 0.7880907456080118, + "0.7": 0.7880967259407043, + "0.8": 0.7873489856719971, + "0.9": 0.7872085372606913 + }, + "matched_cos_abs_band": 0.7876862486203511, + "matched_cos_abs_band_std": 0.0011073626220444157, + "unique_frac_abs_per_threshold": { + "0.5": 0.9583333333333334, + "0.6": 0.9691840277777777, + "0.7": 0.9724392361111112, + "0.8": 0.9753689236111112, + "0.9": 0.9794921875 + }, + "unique_frac_abs_band": 0.97412109375, + "unique_frac_abs_band_std": 0.000281909311127779, + "matched_frac_abs_per_threshold": { + "0.5": 0.8051215277777777, + "0.6": 0.7740885416666666, + "0.7": 0.7252604166666666, + "0.8": 0.6507161458333334, + "0.9": 0.49555121527777773 + }, + "matched_frac_abs_band": 0.661404079861111, + "matched_frac_abs_band_std": 0.0012243146805985877, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 3072 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed0/config.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8661b7883db63a1f6d2922a16b302ff8a496f0ce --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..c932f137d4993f7f5a4de14755a281605fbebd55 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.12594956054687, + "l0_per_token_0.001": 64.12594956054687, + "l0_per_token_0.01": 64.12594956054687, + "l0_per_token_0.05": 64.12594956054687, + "l0_per_token_0.1": 64.12594956054687, + "l0_per_token_0.5": 61.71634790039062, + "l0_per_token_1.0": 22.29517177734375, + "l1_per_token": 80.75087763596787, + "r2_score": 0.9199404659849154, + "l2_loss": 0.0533013150542418 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed1/config.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce45a08b6913db69f2d2155ef8d8951ada06ad6 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..17f070e42a438fbe1565419e8a1e93258b7849ba --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.47314228515626, + "l0_per_token_0.001": 64.47314228515626, + "l0_per_token_0.01": 64.47314228515626, + "l0_per_token_0.05": 64.47314228515626, + "l0_per_token_0.1": 64.47314228515626, + "l0_per_token_0.5": 61.98799604492188, + "l0_per_token_1.0": 22.36115703125, + "l1_per_token": 81.03433132431691, + "r2_score": 0.9197213487439466, + "l2_loss": 0.05344719695623353 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed2/config.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..975d6bca46c0174ff9e6312f6bc2a9da92822fa8 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..f76654474d28ef51d989e4f56ca850a34bd4c499 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.09163022460937, + "l0_per_token_0.001": 64.09163022460937, + "l0_per_token_0.01": 64.09163022460937, + "l0_per_token_0.05": 64.09163022460937, + "l0_per_token_0.1": 64.09163022460937, + "l0_per_token_0.5": 61.66830737304687, + "l0_per_token_1.0": 22.37461181640625, + "l1_per_token": 80.53026138534722, + "r2_score": 0.9458493843498781, + "l2_loss": 0.05331265829289397 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_128/code_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..44db3c2658cd58204d83d608c6909d52a68840fe --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 128.47852833333334, + "recon_r2": 0.9498008531578052, + "l1_per_token": 113.7371251372541 + }, + "seed1": { + "l0": 128.88127333333333, + "recon_r2": 0.9494522496429278, + "l1_per_token": 113.90515653087536 + }, + "seed2": { + "l0": 128.38004166666667, + "recon_r2": 0.949880131442499, + "l1_per_token": 113.08475225154996 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.66681593179832, + "mcc_abs": 0.6668612209253649, + "act_iou": 0.3493809863309316, + "act_iou_n_pairs": 6144, + "input_r2": 0.9669268335458532, + "matched_mcc_signed": { + "0.6": 0.8507408649879782, + "0.7": 0.8758882987674125, + "0.8": 0.9055163254641463, + "0.9": 0.9383849742650251 + }, + "matched_mcc_abs": { + "0.6": 0.8516399976501473, + "0.7": 0.8776236600923026, + "0.8": 0.907436774436553, + "0.9": 0.9391321598878596 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.667911188130844, + "mcc_abs": 0.6679386275423994, + "act_iou": 0.350952910794289, + "act_iou_n_pairs": 6144, + "input_r2": 0.9670698116441203, + "matched_mcc_signed": { + "0.6": 0.8526831815935656, + "0.7": 0.8793382852274035, + "0.8": 0.9086207868302162, + "0.9": 0.9407984977908973 + }, + "matched_mcc_abs": { + "0.6": 0.8542215293298578, + "0.7": 0.8816375294891787, + "0.8": 0.9098555202827411, + "0.9": 0.9413394879983205 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6643578839493046, + "mcc_abs": 0.6643909878433919, + "act_iou": 0.34685079248200507, + "act_iou_n_pairs": 6144, + "input_r2": 0.9668826830669769, + "matched_mcc_signed": { + "0.6": 0.8493503967235192, + "0.7": 0.8780938962284252, + "0.8": 0.9060607405682188, + "0.9": 0.9377283961598311 + }, + "matched_mcc_abs": { + "0.6": 0.8510052847705943, + "0.7": 0.8795877792280521, + "0.8": 0.9075828362296161, + "0.9": 0.9385363459879318 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 128.5799477777778, + "l0_std": 0.2655611487038752, + "recon_r2_mean": 0.9497110780810774, + "recon_r2_std": 0.00022762992312564236, + "l1_per_token_mean": 113.57567797322646, + "l1_per_token_std": 0.4333759170319461, + "input_r2_mean": 0.9669597760856501, + "input_r2_std": 9.781709611379675e-05, + "mcc_signed_mean": 0.6663616679594896, + "mcc_signed_std": 0.0018196866153511505, + "mcc_abs_mean": 0.6663969454370521, + "mcc_abs_std": 0.0018188184778255426, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8509248144350211, + "0.7": 0.877773493407747, + "0.8": 0.9067326176208604, + "0.9": 0.9389706227385846 + }, + "matched_mcc_signed_band": 0.8936003870505533, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8522889372501998, + "0.7": 0.8796163229365112, + "0.8": 0.9082917103163034, + "0.9": 0.9396693312913706 + }, + "matched_mcc_abs_band": 0.8949665754485963, + "act_iou_mean": 0.34906156320240855, + "act_iou_std": 0.002069629680225509, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_128/dict_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..0f935c2f6bdb3beb49665ecfa09918941d88eb50 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 128, + "l0_raw": 128.23066865234375, + "recon_r2": 0.9499843059635639, + "l1_per_token": 113.44417398165874, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 337, + "dense_rate": 0.054850260416666664, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 104, + "0.6": 23, + "0.7": 5, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 50, + "0.6": 11, + "0.7": 2, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 54, + "0.6": 12, + "0.7": 3, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 5, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.2507411539554596, + "n_dup_abs": { + "0.5": 992, + "0.6": 863, + "0.7": 795, + "0.8": 728, + "0.9": 622 + }, + "n_dup_clusters_abs": { + "0.5": 494, + "0.6": 431, + "0.7": 397, + "0.8": 364, + "0.9": 311 + }, + "n_dup_removed_abs": { + "0.5": 498, + "0.6": 432, + "0.7": 398, + "0.8": 364, + "0.9": 311 + }, + "largest_dup_abs": { + "0.5": 5, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3750635087490082, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 129, + "l0_raw": 128.58340571289062, + "recon_r2": 0.9496840996031325, + "l1_per_token": 113.57454236329976, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 341, + "dense_rate": 0.055501302083333336, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 92, + "0.6": 25, + "0.7": 7, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 45, + "0.6": 12, + "0.7": 3, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 47, + "0.6": 13, + "0.7": 4, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.25009188055992126, + "n_dup_abs": { + "0.5": 986, + "0.6": 847, + "0.7": 789, + "0.8": 726, + "0.9": 636 + }, + "n_dup_clusters_abs": { + "0.5": 492, + "0.6": 423, + "0.7": 394, + "0.8": 363, + "0.9": 318 + }, + "n_dup_removed_abs": { + "0.5": 494, + "0.6": 424, + "0.7": 395, + "0.8": 363, + "0.9": 318 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3746306002140045, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 128, + "l0_raw": 128.09353349609376, + "recon_r2": 0.9662462523250043, + "l1_per_token": 112.78520772672366, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 341, + "dense_rate": 0.055501302083333336, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 110, + "0.6": 30, + "0.7": 8, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 51, + "0.6": 14, + "0.7": 3, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 59, + "0.6": 16, + "0.7": 5, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 6, + "0.6": 4, + "0.7": 4, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.2509503662586212, + "n_dup_abs": { + "0.5": 994, + "0.6": 866, + "0.7": 814, + "0.8": 756, + "0.9": 648 + }, + "n_dup_clusters_abs": { + "0.5": 493, + "0.6": 432, + "0.7": 406, + "0.8": 378, + "0.9": 324 + }, + "n_dup_removed_abs": { + "0.5": 501, + "0.6": 434, + "0.7": 408, + "0.8": 378, + "0.9": 324 + }, + "largest_dup_abs": { + "0.5": 6, + "0.6": 4, + "0.7": 4, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3766467869281769, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.687640368938446, + "dcs_abs": 0.6894852519035339, + "frac_sign_flipped": 0.0380859375, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6859203577041626, + "0.6": 0.6875070929527283, + "0.7": 0.6876614093780518, + "0.8": 0.6876947283744812, + "0.9": 0.687640368938446 + }, + "unique_frac_signed": { + "0.5": 0.9917805989583334, + "0.6": 0.9979654947916666, + "0.7": 0.9994303385416666, + "0.8": 0.9999186197916666, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6802825032015487, + "0.6": 0.6861083561913498, + "0.7": 0.6872696751767459, + "0.8": 0.6876387636342163, + "0.9": 0.687640368938446 + }, + "matched_frac_signed": { + "0.5": 0.7307942708333334, + "0.6": 0.67578125, + "0.7": 0.6004231770833334, + "0.8": 0.49072265625, + "0.9": 0.2921549479166667 + }, + "matched_cos_abs": { + "0.5": 0.6995339393615723, + "0.6": 0.6998834609985352, + "0.7": 0.6984584331512451, + "0.8": 0.696729302406311, + "0.9": 0.6959211826324463 + }, + "unique_frac_abs": { + "0.5": 0.9192708333333334, + "0.6": 0.9303385416666666, + "0.7": 0.9354654947916666, + "0.8": 0.9408365885416666, + "0.9": 0.9488118489583334 + }, + "dedup_dcs_abs": { + "0.5": 0.643061147381862, + "0.6": 0.6511285584419966, + "0.7": 0.6533837637592417, + "0.8": 0.6555084200129688, + "0.9": 0.6602982640227614 + }, + "matched_frac_abs": { + "0.5": 0.6966145833333334, + "0.6": 0.6481119791666666, + "0.7": 0.5764973958333334, + "0.8": 0.4713541666666667, + "0.9": 0.2820638020833333 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.6862098574638367, + "dcs_abs": 0.6875536441802979, + "frac_sign_flipped": 0.033203125, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6835977435112, + "0.6": 0.6856574416160583, + "0.7": 0.6862263083457947, + "0.8": 0.686227023601532, + "0.9": 0.6862098574638367 + }, + "unique_frac_signed": { + "0.5": 0.9908040364583334, + "0.6": 0.9977213541666666, + "0.7": 0.9993489583333334, + "0.8": 0.9999186197916666, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6773114035847054, + "0.6": 0.6840950711436259, + "0.7": 0.6857795464262988, + "0.8": 0.6861711783033873, + "0.9": 0.6862098574638367 + }, + "matched_frac_signed": { + "0.5": 0.7283528645833334, + "0.6": 0.67138671875, + "0.7": 0.5909830729166666, + "0.8": 0.48486328125, + "0.9": 0.2869466145833333 + }, + "matched_cos_abs": { + "0.5": 0.6961320638656616, + "0.6": 0.6964331865310669, + "0.7": 0.6966070532798767, + "0.8": 0.6954208016395569, + "0.9": 0.6941182017326355 + }, + "unique_frac_abs": { + "0.5": 0.918701171875, + "0.6": 0.9295247395833334, + "0.7": 0.9344075520833334, + "0.8": 0.9396158854166666, + "0.9": 0.9483235677083334 + }, + "dedup_dcs_abs": { + "0.5": 0.6395373428531457, + "0.6": 0.647351876347481, + "0.7": 0.6509148914192338, + "0.8": 0.6534284322697204, + "0.9": 0.6582486494783856 + }, + "matched_frac_abs": { + "0.5": 0.6925455729166666, + "0.6": 0.6414388020833334, + "0.7": 0.5667317708333334, + "0.8": 0.46728515625, + "0.9": 0.2770182291666667 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.6843628883361816, + "dcs_abs": 0.6857478022575378, + "frac_sign_flipped": 0.037760416666666664, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6827351450920105, + "0.6": 0.6841185092926025, + "0.7": 0.6844413876533508, + "0.8": 0.6843628883361816, + "0.9": 0.6843628883361816 + }, + "unique_frac_signed": { + "0.5": 0.9913736979166666, + "0.6": 0.9976399739583334, + "0.7": 0.999267578125, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6768456654875383, + "0.6": 0.6825039717950858, + "0.7": 0.6839400878088782, + "0.8": 0.6843628883361816, + "0.9": 0.6843628883361816 + }, + "matched_frac_signed": { + "0.5": 0.73388671875, + "0.6": 0.6671549479166666, + "0.7": 0.587890625, + "0.8": 0.4773763020833333, + "0.9": 0.2841796875 + }, + "matched_cos_abs": { + "0.5": 0.69392329454422, + "0.6": 0.6943584084510803, + "0.7": 0.6939539313316345, + "0.8": 0.6927098631858826, + "0.9": 0.6908969879150391 + }, + "unique_frac_abs": { + "0.5": 0.9190266927083334, + "0.6": 0.93017578125, + "0.7": 0.9346516927083334, + "0.8": 0.939697265625, + "0.9": 0.94775390625 + }, + "dedup_dcs_abs": { + "0.5": 0.6377340303782452, + "0.6": 0.6458753750484902, + "0.7": 0.6486052165807148, + "0.8": 0.6509375643072417, + "0.9": 0.6548003191128373 + }, + "matched_frac_abs": { + "0.5": 0.6979166666666666, + "0.6": 0.6365559895833334, + "0.7": 0.5625, + "0.8": 0.4567057291666667, + "0.9": 0.2734375 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 340, + "n_dense_std": 2.309401076758503, + "l0_mean": 128, + "l0_std": 0.25271996153596465, + "recon_r2_mean": 0.9553048859639003, + "recon_r2_std": 0.009476690052472725, + "l1_per_token_mean": 113.2679746905607, + "l1_per_token_std": 0.42313937982922933, + "babel_mu1_signed_mean": 0.25059446692466736, + "babel_mu1_signed_std": 0.00044764633079035475, + "n_dup_signed_per_threshold": { + "0.5": 102.0, + "0.6": 26.0, + "0.7": 6.666666666666667, + "0.8": 0.6666666666666666, + "0.9": 0.0 + }, + "n_dup_signed_band": 8.333333333333332, + "n_dup_signed_band_std": 1.0408329997330665, + "n_dup_clusters_signed_per_threshold": { + "0.5": 48.666666666666664, + "0.6": 12.333333333333334, + "0.7": 2.6666666666666665, + "0.8": 0.3333333333333333, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 3.8333333333333335, + "n_dup_clusters_signed_band_std": 0.3818813079129867, + "largest_dup_signed_per_threshold": { + "0.5": 4.666666666666667, + "0.6": 3.3333333333333335, + "0.7": 3.3333333333333335, + "0.8": 1.3333333333333333, + "0.9": 1.0 + }, + "largest_dup_signed_band": 2.25, + "largest_dup_signed_band_std": 0.25, + "dcs_signed_mean": 0.6860710382461548, + "dcs_signed_std": 0.001643144198161932, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.6781465240912642, + "0.6": 0.6842357997100205, + "0.7": 0.6856631031373076, + "0.8": 0.6860576100912618, + "0.9": 0.6860710382461548 + }, + "dedup_dcs_signed_band": 0.6855068877961862, + "dedup_dcs_signed_band_std": 0.0016866391290425423, + "matched_cos_signed_per_threshold": { + "0.5": 0.684084415435791, + "0.6": 0.685761014620463, + "0.7": 0.686109701792399, + "0.8": 0.6860948801040649, + "0.9": 0.6860710382461548 + }, + "matched_cos_signed_band": 0.6860091586907704, + "matched_cos_signed_band_std": 0.0016533844552861247, + "unique_frac_signed_per_threshold": { + "0.5": 0.9913194444444445, + "0.6": 0.9977756076388888, + "0.7": 0.9993489583333334, + "0.8": 0.9999457465277777, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.999267578125, + "unique_frac_signed_band_std": 5.3827948223149554e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.7310112847222223, + "0.6": 0.6714409722222222, + "0.7": 0.5930989583333334, + "0.8": 0.48432074652777773, + "0.9": 0.2877604166666667 + }, + "matched_frac_signed_band": 0.5091552734375, + "matched_frac_signed_band_std": 0.005336302000584569, + "babel_mu1_abs_mean": 0.3754469652970632, + "babel_mu1_abs_std": 0.0010613818397664375, + "n_dup_abs_per_threshold": { + "0.5": 990.6666666666666, + "0.6": 858.6666666666666, + "0.7": 799.3333333333334, + "0.8": 736.6666666666666, + "0.9": 635.3333333333334 + }, + "n_dup_abs_band": 757.5, + "n_dup_abs_band_std": 11.757976016304847, + "n_dup_clusters_abs_per_threshold": { + "0.5": 493.0, + "0.6": 428.6666666666667, + "0.7": 399.0, + "0.8": 368.3333333333333, + "0.9": 317.6666666666667 + }, + "n_dup_clusters_abs_band": 378.4166666666667, + "n_dup_clusters_abs_band_std": 5.735488935856588, + "largest_dup_abs_per_threshold": { + "0.5": 4.666666666666667, + "0.6": 3.3333333333333335, + "0.7": 3.3333333333333335, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.666666666666667, + "largest_dup_abs_band_std": 0.2886751345948129, + "dcs_abs_mean": 0.6875955661137899, + "dcs_abs_std": 0.001869077459729763, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6401108402044177, + "0.6": 0.6481186032793226, + "0.7": 0.6509679572530634, + "0.8": 0.6532914721966436, + "0.9": 0.657782410871328 + }, + "dedup_dcs_abs_band": 0.6525401109000895, + "dedup_dcs_abs_band_std": 0.002513003970042433, + "matched_cos_abs_per_threshold": { + "0.5": 0.696529765923818, + "0.6": 0.6968916853268942, + "0.7": 0.6963398059209188, + "0.8": 0.6949533224105835, + "0.9": 0.6936454574267069 + }, + "matched_cos_abs_band": 0.6954575677712759, + "matched_cos_abs_band_std": 0.0023896567147719903, + "unique_frac_abs_per_threshold": { + "0.5": 0.9189995659722223, + "0.6": 0.9300130208333334, + "0.7": 0.9348415798611112, + "0.8": 0.9400499131944443, + "0.9": 0.9482964409722223 + }, + "unique_frac_abs_band": 0.9383002387152778, + "unique_frac_abs_band_std": 0.00049011451457124, + "matched_frac_abs_per_threshold": { + "0.5": 0.6956922743055555, + "0.6": 0.6420355902777778, + "0.7": 0.568576388888889, + "0.8": 0.46511501736111116, + "0.9": 0.2775065104166667 + }, + "matched_frac_abs_band": 0.48830837673611116, + "matched_frac_abs_band_std": 0.006105730573216553, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 6144 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed0/config.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..800c0ac8a2bdf4ea9d19bd42bae6c63a2420466c --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b30fd0eaa9aad626efb6c3df5d950727e49c5abd --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.23066865234375, + "l0_per_token_0.001": 128.23066865234375, + "l0_per_token_0.01": 128.23066865234375, + "l0_per_token_0.05": 128.23066865234375, + "l0_per_token_0.1": 128.23066865234375, + "l0_per_token_0.5": 77.84084619140626, + "l0_per_token_1.0": 25.73614052734375, + "l1_per_token": 113.44417398165874, + "r2_score": 0.9499843059635639, + "l2_loss": 0.033298998030519895 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed1/config.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b9fbd7a21b669bb2b7076cc96b17e6cfe86e4d69 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b74eaaca4c4c2e256e5ff3012b571cff563f49e5 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.58340571289062, + "l0_per_token_0.001": 128.58340571289062, + "l0_per_token_0.01": 128.58340571289062, + "l0_per_token_0.05": 128.58340571289062, + "l0_per_token_0.1": 128.58340571289062, + "l0_per_token_0.5": 77.9844111328125, + "l0_per_token_1.0": 25.639007373046876, + "l1_per_token": 113.57454236329976, + "r2_score": 0.9496840996031325, + "l2_loss": 0.03349886671568643 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed2/config.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..08c771d30493c2f267b68028023ab28a4e62bd16 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 128, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..8bd1b627049e4f778bbb96b9aa2ffa12d07a5c8f --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.09353349609376, + "l0_per_token_0.001": 128.09353349609376, + "l0_per_token_0.01": 128.09353349609376, + "l0_per_token_0.05": 128.09353349609376, + "l0_per_token_0.1": 128.09353349609376, + "l0_per_token_0.5": 77.62712329101562, + "l0_per_token_1.0": 25.453875537109376, + "l1_per_token": 112.78520772672366, + "r2_score": 0.9662462523250043, + "l2_loss": 0.033231423028106666 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_32/code_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..fbaa846a46dbbfcee6058aea76887b99358deb8f --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 32.088645, + "recon_r2": 0.905954283630278, + "l1_per_token": 58.9126479012386 + }, + "seed1": { + "l0": 32.316028333333335, + "recon_r2": 0.9058372732971369, + "l1_per_token": 59.16793039634953 + }, + "seed2": { + "l0": 32.161475, + "recon_r2": 0.9060133827296776, + "l1_per_token": 58.996684597208 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7506348134037194, + "mcc_abs": 0.7506489559851172, + "act_iou": 0.45286836887413534, + "act_iou_n_pairs": 6144, + "input_r2": 0.9676482017337845, + "matched_mcc_signed": { + "0.6": 0.8549533515109892, + "0.7": 0.8772884914888265, + "0.8": 0.9019156925335778, + "0.9": 0.9306478196088455 + }, + "matched_mcc_abs": { + "0.6": 0.8553181617001057, + "0.7": 0.877124628041597, + "0.8": 0.9014061661097836, + "0.9": 0.9305727391785101 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7547558716520336, + "mcc_abs": 0.7547696342375708, + "act_iou": 0.4575580039974237, + "act_iou_n_pairs": 6144, + "input_r2": 0.9684256811250374, + "matched_mcc_signed": { + "0.6": 0.8564369804360407, + "0.7": 0.8806091524623597, + "0.8": 0.904498173539956, + "0.9": 0.9312682265634307 + }, + "matched_mcc_abs": { + "0.6": 0.8559008938153212, + "0.7": 0.880266445790391, + "0.8": 0.9042910297663749, + "0.9": 0.9312158239703362 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7535905263861533, + "mcc_abs": 0.7536051346631402, + "act_iou": 0.45511378587286294, + "act_iou_n_pairs": 6144, + "input_r2": 0.9693433634830741, + "matched_mcc_signed": { + "0.6": 0.8537058669181209, + "0.7": 0.877944197296065, + "0.8": 0.9027274560057406, + "0.9": 0.9283330211242621 + }, + "matched_mcc_abs": { + "0.6": 0.8530305791536428, + "0.7": 0.8774386707513464, + "0.8": 0.9027692890311738, + "0.9": 0.928081154141351 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 32.18871611111111, + "l0_std": 0.11611353784279671, + "recon_r2_mean": 0.9059349798856974, + "recon_r2_std": 8.962761278517184e-05, + "l1_per_token_mean": 59.02575429826538, + "l1_per_token_std": 0.1301002448775166, + "input_r2_mean": 0.9684724154472987, + "input_r2_std": 0.0008485466467564345, + "mcc_signed_mean": 0.752993737147302, + "mcc_signed_std": 0.0021243583309333907, + "mcc_abs_mean": 0.7530079082952761, + "mcc_abs_std": 0.002124266182924178, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8550320662883836, + "0.7": 0.878613947082417, + "0.8": 0.903047107359758, + "0.9": 0.9300830224321794 + }, + "matched_mcc_signed_band": 0.8916940357906845, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8547498782230232, + "0.7": 0.8782765815277781, + "0.8": 0.9028221616357776, + "0.9": 0.9299565724300658 + }, + "matched_mcc_abs_band": 0.8914512984541612, + "act_iou_mean": 0.45518005291480734, + "act_iou_std": 0.0023455197479516085, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_32/dict_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..f1d7f47ce8d293360bf0ed2aa98dc9b62137a3fd --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 32, + "l0_raw": 31.99159658203125, + "recon_r2": 0.9063443286947883, + "l1_per_token": 58.74865446735893, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 21, + "dense_rate": 0.00341796875, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 541, + "0.6": 155, + "0.7": 28, + "0.8": 5, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 254, + "0.6": 72, + "0.7": 11, + "0.8": 2, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 287, + "0.6": 83, + "0.7": 17, + "0.8": 3, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 10, + "0.6": 10, + "0.7": 7, + "0.8": 3, + "0.9": 1 + }, + "babel_mu1_signed": 0.36175426840782166, + "n_dup_abs": { + "0.5": 683, + "0.6": 271, + "0.7": 140, + "0.8": 83, + "0.9": 66 + }, + "n_dup_clusters_abs": { + "0.5": 325, + "0.6": 130, + "0.7": 67, + "0.8": 41, + "0.9": 33 + }, + "n_dup_removed_abs": { + "0.5": 358, + "0.6": 141, + "0.7": 73, + "0.8": 42, + "0.9": 33 + }, + "largest_dup_abs": { + "0.5": 10, + "0.6": 10, + "0.7": 7, + "0.8": 3, + "0.9": 2 + }, + "babel_mu1_abs": 0.3797977864742279, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 32, + "l0_raw": 32.19938203125, + "recon_r2": 0.9062973391016057, + "l1_per_token": 58.98012959271581, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 21, + "dense_rate": 0.00341796875, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 529, + "0.6": 126, + "0.7": 24, + "0.8": 5, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 254, + "0.6": 61, + "0.7": 11, + "0.8": 2, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 275, + "0.6": 65, + "0.7": 13, + "0.8": 3, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 5, + "0.6": 5, + "0.7": 4, + "0.8": 3, + "0.9": 1 + }, + "babel_mu1_signed": 0.36123380064964294, + "n_dup_abs": { + "0.5": 676, + "0.6": 234, + "0.7": 116, + "0.8": 87, + "0.9": 56 + }, + "n_dup_clusters_abs": { + "0.5": 327, + "0.6": 115, + "0.7": 57, + "0.8": 43, + "0.9": 28 + }, + "n_dup_removed_abs": { + "0.5": 349, + "0.6": 119, + "0.7": 59, + "0.8": 44, + "0.9": 28 + }, + "largest_dup_abs": { + "0.5": 5, + "0.6": 5, + "0.7": 4, + "0.8": 3, + "0.9": 2 + }, + "babel_mu1_abs": 0.3785272538661957, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 32, + "l0_raw": 32.05425063476562, + "recon_r2": 0.936707125907676, + "l1_per_token": 58.812589024485604, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 21, + "dense_rate": 0.00341796875, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 570, + "0.6": 154, + "0.7": 27, + "0.8": 3, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 273, + "0.6": 72, + "0.7": 12, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 297, + "0.6": 82, + "0.7": 15, + "0.8": 2, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 7, + "0.6": 7, + "0.7": 5, + "0.8": 3, + "0.9": 1 + }, + "babel_mu1_signed": 0.361777663230896, + "n_dup_abs": { + "0.5": 696, + "0.6": 262, + "0.7": 117, + "0.8": 83, + "0.9": 58 + }, + "n_dup_clusters_abs": { + "0.5": 336, + "0.6": 126, + "0.7": 57, + "0.8": 41, + "0.9": 29 + }, + "n_dup_removed_abs": { + "0.5": 360, + "0.6": 136, + "0.7": 60, + "0.8": 42, + "0.9": 29 + }, + "largest_dup_abs": { + "0.5": 7, + "0.6": 7, + "0.7": 5, + "0.8": 3, + "0.9": 2 + }, + "babel_mu1_abs": 0.3785177767276764, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8151049017906189, + "dcs_abs": 0.81593257188797, + "frac_sign_flipped": 0.015299479166666666, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8072613477706909, + "0.6": 0.8140349984169006, + "0.7": 0.8151996731758118, + "0.8": 0.8151456117630005, + "0.9": 0.8151049017906189 + }, + "unique_frac_signed": { + "0.5": 0.9542643229166666, + "0.6": 0.9879557291666666, + "0.7": 0.99755859375, + "0.8": 0.99951171875, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7703407034471941, + "0.6": 0.8042305404281553, + "0.7": 0.8132094395987224, + "0.8": 0.8147475914447568, + "0.9": 0.8151049017906189 + }, + "matched_frac_signed": { + "0.5": 0.8181966145833334, + "0.6": 0.8175455729166666, + "0.7": 0.7828776041666666, + "0.8": 0.7057291666666666, + "0.9": 0.5603841145833334 + }, + "matched_cos_abs": { + "0.5": 0.8094140887260437, + "0.6": 0.8165645599365234, + "0.7": 0.817976713180542, + "0.8": 0.8171650767326355, + "0.9": 0.8169581890106201 + }, + "unique_frac_abs": { + "0.5": 0.9424641927083334, + "0.6": 0.9788411458333334, + "0.7": 0.9892578125, + "0.8": 0.9930013020833334, + "0.9": 0.9950358072916666 + }, + "dedup_dcs_abs": { + "0.5": 0.7628437956979421, + "0.6": 0.7992869894951582, + "0.7": 0.8091898539569229, + "0.8": 0.8114459852125341, + "0.9": 0.8129026511257204 + }, + "matched_frac_abs": { + "0.5": 0.8092447916666666, + "0.6": 0.8121744791666666, + "0.7": 0.7801106770833334, + "0.8": 0.7054036458333334, + "0.9": 0.5594075520833334 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.817715585231781, + "dcs_abs": 0.8186044096946716, + "frac_sign_flipped": 0.011393229166666666, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8107132315635681, + "0.6": 0.8155848383903503, + "0.7": 0.8176776766777039, + "0.8": 0.8178020119667053, + "0.9": 0.817715585231781 + }, + "unique_frac_signed": { + "0.5": 0.9524739583333334, + "0.6": 0.986572265625, + "0.7": 0.9973958333333334, + "0.8": 0.9995930989583334, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.77218324074056, + "0.6": 0.8046333818201674, + "0.7": 0.8155483077280223, + "0.8": 0.817469247476159, + "0.9": 0.817715585231781 + }, + "matched_frac_signed": { + "0.5": 0.8201497395833334, + "0.6": 0.8191731770833334, + "0.7": 0.78173828125, + "0.8": 0.7060546875, + "0.9": 0.57275390625 + }, + "matched_cos_abs": { + "0.5": 0.8123207092285156, + "0.6": 0.8180883526802063, + "0.7": 0.8203215599060059, + "0.8": 0.8196021914482117, + "0.9": 0.8196630477905273 + }, + "unique_frac_abs": { + "0.5": 0.9415690104166666, + "0.6": 0.9774576822916666, + "0.7": 0.9891764322916666, + "0.8": 0.9931640625, + "0.9": 0.9949544270833334 + }, + "dedup_dcs_abs": { + "0.5": 0.7648560063292582, + "0.6": 0.799646745120602, + "0.7": 0.8114427539597575, + "0.8": 0.8139994420926087, + "0.9": 0.815527378115803 + }, + "matched_frac_abs": { + "0.5": 0.8126627604166666, + "0.6": 0.8152669270833334, + "0.7": 0.7789713541666666, + "0.8": 0.70556640625, + "0.9": 0.57177734375 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8174644112586975, + "dcs_abs": 0.8188273310661316, + "frac_sign_flipped": 0.014973958333333334, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8102830052375793, + "0.6": 0.8165702223777771, + "0.7": 0.8172942399978638, + "0.8": 0.8174680471420288, + "0.9": 0.8174643516540527 + }, + "unique_frac_signed": { + "0.5": 0.9534505208333334, + "0.6": 0.988037109375, + "0.7": 0.9977213541666666, + "0.8": 0.9995930989583334, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7725647533661686, + "0.6": 0.8068016821198398, + "0.7": 0.8154319158832852, + "0.8": 0.8171354185421176, + "0.9": 0.8174643516540527 + }, + "matched_frac_signed": { + "0.5": 0.8194986979166666, + "0.6": 0.8248697916666666, + "0.7": 0.78466796875, + "0.8": 0.705078125, + "0.9": 0.5686848958333334 + }, + "matched_cos_abs": { + "0.5": 0.8116031289100647, + "0.6": 0.8187584280967712, + "0.7": 0.819368839263916, + "0.8": 0.8196461200714111, + "0.9": 0.8191943764686584 + }, + "unique_frac_abs": { + "0.5": 0.9423014322916666, + "0.6": 0.979248046875, + "0.7": 0.9903157552083334, + "0.8": 0.9930013020833334, + "0.9": 0.995361328125 + }, + "dedup_dcs_abs": { + "0.5": 0.7647747908243521, + "0.6": 0.8017675915762084, + "0.7": 0.8114338708498205, + "0.8": 0.8139096644784635, + "0.9": 0.8153944025543751 + }, + "matched_frac_abs": { + "0.5": 0.8115234375, + "0.6": 0.8199869791666666, + "0.7": 0.7828776041666666, + "0.8": 0.70361328125, + "0.9": 0.5670572916666666 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 21, + "n_dense_std": 0.0, + "l0_mean": 32, + "l0_std": 0.10658599460575449, + "recon_r2_mean": 0.9164495979013566, + "recon_r2_std": 0.017543549603788174, + "l1_per_token_mean": 58.847124361520116, + "l1_per_token_std": 0.11953953548015549, + "babel_mu1_signed_mean": 0.36158857742945355, + "babel_mu1_signed_std": 0.00030746829439187305, + "n_dup_signed_per_threshold": { + "0.5": 546.6666666666666, + "0.6": 145.0, + "0.7": 26.333333333333332, + "0.8": 4.333333333333333, + "0.9": 0.0 + }, + "n_dup_signed_band": 43.91666666666667, + "n_dup_signed_band_std": 4.502314219746699, + "n_dup_clusters_signed_per_threshold": { + "0.5": 260.3333333333333, + "0.6": 68.33333333333333, + "0.7": 11.333333333333334, + "0.8": 1.6666666666666667, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 20.333333333333332, + "n_dup_clusters_signed_band_std": 1.5877132402714706, + "largest_dup_signed_per_threshold": { + "0.5": 7.333333333333333, + "0.6": 7.333333333333333, + "0.7": 5.333333333333333, + "0.8": 3.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 4.166666666666666, + "largest_dup_signed_band_std": 1.0103629710818451, + "dcs_signed_mean": 0.8167616327603658, + "dcs_signed_std": 0.0014402569981699564, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7716962325179741, + "0.6": 0.8052218681227208, + "0.7": 0.8147298877366765, + "0.8": 0.8164507524876777, + "0.9": 0.8167616128921509 + }, + "dedup_dcs_signed_band": 0.8132910303098064, + "dedup_dcs_signed_band_std": 0.0012844039651080125, + "matched_cos_signed_per_threshold": { + "0.5": 0.8094191948572794, + "0.6": 0.8153966863950094, + "0.7": 0.8167238632837931, + "0.8": 0.8168052236239115, + "0.9": 0.8167616128921509 + }, + "matched_cos_signed_band": 0.8164218465487162, + "matched_cos_signed_band_std": 0.0013428175489496707, + "unique_frac_signed_per_threshold": { + "0.5": 0.9533962673611112, + "0.6": 0.9875217013888888, + "0.7": 0.99755859375, + "0.8": 0.9995659722222223, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9961615668402778, + "unique_frac_signed_band_std": 0.0002384222524853963, + "matched_frac_signed_per_threshold": { + "0.5": 0.8192816840277778, + "0.6": 0.8205295138888888, + "0.7": 0.7830946180555555, + "0.8": 0.7056206597222222, + "0.9": 0.5672743055555557 + }, + "matched_frac_signed_band": 0.7191297743055556, + "matched_frac_signed_band_std": 0.002207164679100782, + "babel_mu1_abs_mean": 0.37894760568936664, + "babel_mu1_abs_std": 0.0007362934056808417, + "n_dup_abs_per_threshold": { + "0.5": 685.0, + "0.6": 255.66666666666666, + "0.7": 124.33333333333333, + "0.8": 84.33333333333333, + "0.9": 60.0 + }, + "n_dup_abs_band": 131.08333333333331, + "n_dup_abs_band_std": 8.427385913397663, + "n_dup_clusters_abs_per_threshold": { + "0.5": 329.3333333333333, + "0.6": 123.66666666666667, + "0.7": 60.333333333333336, + "0.8": 41.666666666666664, + "0.9": 30.0 + }, + "n_dup_clusters_abs_band": 63.916666666666664, + "n_dup_clusters_abs_band_std": 3.547299442298794, + "largest_dup_abs_per_threshold": { + "0.5": 7.333333333333333, + "0.6": 7.333333333333333, + "0.7": 5.333333333333333, + "0.8": 3.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 4.416666666666666, + "largest_dup_abs_band_std": 1.0103629710818451, + "dcs_abs_mean": 0.8177881042162577, + "dcs_abs_std": 0.0016107990720304458, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.7641581976171841, + "0.6": 0.8002337753973228, + "0.7": 0.8106888262555003, + "0.8": 0.8131183639278688, + "0.9": 0.8146081439319661 + }, + "dedup_dcs_abs_band": 0.8096622773781645, + "dedup_dcs_abs_band_std": 0.0012827771660477818, + "matched_cos_abs_per_threshold": { + "0.5": 0.811112642288208, + "0.6": 0.8178037802378336, + "0.7": 0.8192223707834879, + "0.8": 0.8188044627507528, + "0.9": 0.8186052044232687 + }, + "matched_cos_abs_band": 0.8186089545488358, + "matched_cos_abs_band_std": 0.0012526434122227966, + "unique_frac_abs_per_threshold": { + "0.5": 0.9421115451388888, + "0.6": 0.978515625, + "0.7": 0.9895833333333334, + "0.8": 0.9930555555555557, + "0.9": 0.9951171875 + }, + "unique_frac_abs_band": 0.9890679253472223, + "unique_frac_abs_band_std": 0.0003978138394114712, + "matched_frac_abs_per_threshold": { + "0.5": 0.8111436631944443, + "0.6": 0.8158094618055555, + "0.7": 0.7806532118055555, + "0.8": 0.7048611111111112, + "0.9": 0.5660807291666666 + }, + "matched_frac_abs_band": 0.7168511284722222, + "matched_frac_abs_band_std": 0.002245095932812976, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 6144 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed0/config.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..805139f825bcdfb5940e33cea037c7c7a87efe38 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..3ce221f6aedbcedc198046d816689900abe986c4 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.99159658203125, + "l0_per_token_0.001": 31.99159658203125, + "l0_per_token_0.01": 31.99159658203125, + "l0_per_token_0.05": 31.99159658203125, + "l0_per_token_0.1": 31.99159658203125, + "l0_per_token_0.5": 31.99159658203125, + "l0_per_token_1.0": 18.41407568359375, + "l1_per_token": 58.74865446735893, + "r2_score": 0.9063443286947883, + "l2_loss": 0.06235322880988821 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed1/config.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..80f1abe0c12e221d402f0d112333352fb502d1d0 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e17c4bf6abaee947e0ad4ce57124d53d0af746aa --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.19938203125, + "l0_per_token_0.001": 32.19938203125, + "l0_per_token_0.01": 32.19938203125, + "l0_per_token_0.05": 32.19938203125, + "l0_per_token_0.1": 32.19938203125, + "l0_per_token_0.5": 32.19938203125, + "l0_per_token_1.0": 18.5937828125, + "l1_per_token": 58.98012959271581, + "r2_score": 0.9062973391016057, + "l2_loss": 0.06238451311776376 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed2/config.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ccd49714f71afa3b581cadb1c2e688b8da4a4291 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 32, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..609f4d4630a3c1370d42e74eb29b44eb078b3a34 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.05425063476562, + "l0_per_token_0.001": 32.05425063476562, + "l0_per_token_0.01": 32.05425063476562, + "l0_per_token_0.05": 32.05425063476562, + "l0_per_token_0.1": 32.05425063476562, + "l0_per_token_0.5": 32.05425063476562, + "l0_per_token_1.0": 18.49962734375, + "l1_per_token": 58.812589024485604, + "r2_score": 0.936707125907676, + "l2_loss": 0.06231344423969884 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_64/code_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..2e36279be8ac37c2a4b4fa6f2bbb6fa1ae2ce3ee --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 64.310955, + "recon_r2": 0.9278633690633902, + "l1_per_token": 78.45715137340169 + }, + "seed1": { + "l0": 64.68350166666667, + "recon_r2": 0.927585029138217, + "l1_per_token": 78.70750438633878 + }, + "seed2": { + "l0": 64.21269, + "recon_r2": 0.9277646137550721, + "l1_per_token": 78.27220695803682 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.7119329557196736, + "mcc_abs": 0.7119489692170013, + "act_iou": 0.3954854499495292, + "act_iou_n_pairs": 6144, + "input_r2": 0.9673241245994497, + "matched_mcc_signed": { + "0.6": 0.8493708150341088, + "0.7": 0.8757004699577609, + "0.8": 0.9040849895209604, + "0.9": 0.932686490735146 + }, + "matched_mcc_abs": { + "0.6": 0.849401161827963, + "0.7": 0.8761090127619257, + "0.8": 0.9041459505358026, + "0.9": 0.932680195328028 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7106875993411608, + "mcc_abs": 0.710703678463939, + "act_iou": 0.39463529671558123, + "act_iou_n_pairs": 6144, + "input_r2": 0.9678803105616882, + "matched_mcc_signed": { + "0.6": 0.8519217690775316, + "0.7": 0.876853739812305, + "0.8": 0.902433986723167, + "0.9": 0.9336881204908878 + }, + "matched_mcc_abs": { + "0.6": 0.8517229729757787, + "0.7": 0.8769916564993644, + "0.8": 0.9022079566124955, + "0.9": 0.9334222215195941 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.7118730095397399, + "mcc_abs": 0.711884828802793, + "act_iou": 0.396295442831198, + "act_iou_n_pairs": 6144, + "input_r2": 0.9676076989390286, + "matched_mcc_signed": { + "0.6": 0.8498938738043165, + "0.7": 0.8755544586843508, + "0.8": 0.903109508324468, + "0.9": 0.9330291311768723 + }, + "matched_mcc_abs": { + "0.6": 0.8492480341290426, + "0.7": 0.8755860697484001, + "0.8": 0.9028287301816269, + "0.9": 0.9328731548332438 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 64.40238222222223, + "l0_std": 0.24836487088518894, + "recon_r2_mean": 0.9277376706522263, + "recon_r2_std": 0.000141112460752264, + "l1_per_token_mean": 78.4789542392591, + "l1_per_token_std": 0.21846621361499632, + "input_r2_mean": 0.9676040447000555, + "input_r2_std": 0.00027811098727782306, + "mcc_signed_mean": 0.7114978548668581, + "mcc_signed_std": 0.0007023417250985751, + "mcc_abs_mean": 0.7115124921612445, + "mcc_abs_std": 0.0007011869906775182, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8503954859719857, + "0.7": 0.8760362228181388, + "0.8": 0.9032094948561985, + "0.9": 0.9331345808009687 + }, + "matched_mcc_signed_band": 0.8906939461118228, + "matched_mcc_abs_per_threshold": { + "0.6": 0.850124056310928, + "0.7": 0.87622891300323, + "0.8": 0.903060879109975, + "0.9": 0.9329918572269552 + }, + "matched_mcc_abs_band": 0.8906014264127721, + "act_iou_mean": 0.3954720631654361, + "act_iou_std": 0.0008301540132955006, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_64/dict_metrics_step_final.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..329c36c1b4f3ccc3a0c3d4f3eb5348121f562b8e --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 64.12163701171875, + "recon_r2": 0.9281265595421594, + "l1_per_token": 78.22586933257212, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 78, + "dense_rate": 0.0126953125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 208, + "0.6": 42, + "0.7": 10, + "0.8": 3, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 102, + "0.6": 20, + "0.7": 4, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 106, + "0.6": 22, + "0.7": 6, + "0.8": 2, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 4, + "0.6": 4, + "0.7": 4, + "0.8": 3, + "0.9": 1 + }, + "babel_mu1_signed": 0.29060447216033936, + "n_dup_abs": { + "0.5": 520, + "0.6": 312, + "0.7": 246, + "0.8": 205, + "0.9": 166 + }, + "n_dup_clusters_abs": { + "0.5": 257, + "0.6": 155, + "0.7": 122, + "0.8": 102, + "0.9": 83 + }, + "n_dup_removed_abs": { + "0.5": 263, + "0.6": 157, + "0.7": 124, + "0.8": 103, + "0.9": 83 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 4, + "0.7": 4, + "0.8": 3, + "0.9": 2 + }, + "babel_mu1_abs": 0.33510932326316833, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 64.46833608398437, + "recon_r2": 0.9279219473142333, + "l1_per_token": 78.45223097013593, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 78, + "dense_rate": 0.0126953125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 224, + "0.6": 50, + "0.7": 5, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 109, + "0.6": 24, + "0.7": 2, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 115, + "0.6": 26, + "0.7": 3, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 4, + "0.6": 4, + "0.7": 3, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.29145485162734985, + "n_dup_abs": { + "0.5": 522, + "0.6": 308, + "0.7": 241, + "0.8": 218, + "0.9": 170 + }, + "n_dup_clusters_abs": { + "0.5": 257, + "0.6": 153, + "0.7": 120, + "0.8": 109, + "0.9": 85 + }, + "n_dup_removed_abs": { + "0.5": 265, + "0.6": 155, + "0.7": 121, + "0.8": 109, + "0.9": 85 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 4, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3351520001888275, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 64.0201421875, + "recon_r2": 0.9513525527732973, + "l1_per_token": 78.03207814069725, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 79, + "dense_rate": 0.012858072916666666, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 204, + "0.6": 57, + "0.7": 11, + "0.8": 6, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 97, + "0.6": 25, + "0.7": 4, + "0.8": 2, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 107, + "0.6": 32, + "0.7": 7, + "0.8": 4, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 7, + "0.6": 7, + "0.7": 4, + "0.8": 4, + "0.9": 1 + }, + "babel_mu1_signed": 0.2911992073059082, + "n_dup_abs": { + "0.5": 515, + "0.6": 330, + "0.7": 259, + "0.8": 222, + "0.9": 174 + }, + "n_dup_clusters_abs": { + "0.5": 251, + "0.6": 161, + "0.7": 128, + "0.8": 110, + "0.9": 87 + }, + "n_dup_removed_abs": { + "0.5": 264, + "0.6": 169, + "0.7": 131, + "0.8": 112, + "0.9": 87 + }, + "largest_dup_abs": { + "0.5": 8, + "0.6": 8, + "0.7": 4, + "0.8": 4, + "0.9": 2 + }, + "babel_mu1_abs": 0.3353729546070099, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.757940948009491, + "dcs_abs": 0.7594957947731018, + "frac_sign_flipped": 0.023274739583333332, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.7545055747032166, + "0.6": 0.7571747303009033, + "0.7": 0.7578350901603699, + "0.8": 0.7579107880592346, + "0.9": 0.757940948009491 + }, + "unique_frac_signed": { + "0.5": 0.9820149739583334, + "0.6": 0.99609375, + "0.7": 0.999267578125, + "0.8": 0.999755859375, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7409357722935965, + "0.6": 0.7542170165106654, + "0.7": 0.7572800351626938, + "0.8": 0.7577257512457436, + "0.9": 0.757940948009491 + }, + "matched_frac_signed": { + "0.5": 0.7911783854166666, + "0.6": 0.7618815104166666, + "0.7": 0.69970703125, + "0.8": 0.5994466145833334, + "0.9": 0.4386393229166667 + }, + "matched_cos_abs": { + "0.5": 0.7591702938079834, + "0.6": 0.7612788081169128, + "0.7": 0.7618290781974792, + "0.8": 0.7618495225906372, + "0.9": 0.7611899971961975 + }, + "unique_frac_abs": { + "0.5": 0.95703125, + "0.6": 0.974609375, + "0.7": 0.9800618489583334, + "0.8": 0.9827473958333334, + "0.9": 0.986328125 + }, + "dedup_dcs_abs": { + "0.5": 0.7265496952459216, + "0.6": 0.7419494633795694, + "0.7": 0.7466396149684442, + "0.8": 0.748705634342817, + "0.9": 0.7507831027032807 + }, + "matched_frac_abs": { + "0.5": 0.7779947916666666, + "0.6": 0.7516276041666666, + "0.7": 0.6925455729166666, + "0.8": 0.5940755208333334, + "0.9": 0.4352213541666667 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.7569624781608582, + "dcs_abs": 0.7584924697875977, + "frac_sign_flipped": 0.023763020833333332, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.7531800866127014, + "0.6": 0.7565007209777832, + "0.7": 0.7569905519485474, + "0.8": 0.7571321129798889, + "0.9": 0.7569624781608582 + }, + "unique_frac_signed": { + "0.5": 0.982666015625, + "0.6": 0.99560546875, + "0.7": 0.9989420572916666, + "0.8": 0.99951171875, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7401244747597957, + "0.6": 0.7531762549187988, + "0.7": 0.7561896993138362, + "0.8": 0.756762419565348, + "0.9": 0.7569624781608582 + }, + "matched_frac_signed": { + "0.5": 0.7926432291666666, + "0.6": 0.7547200520833334, + "0.7": 0.6980794270833334, + "0.8": 0.6061197916666666, + "0.9": 0.43603515625 + }, + "matched_cos_abs": { + "0.5": 0.7574422955513, + "0.6": 0.760525643825531, + "0.7": 0.7609390020370483, + "0.8": 0.7608199119567871, + "0.9": 0.7601877450942993 + }, + "unique_frac_abs": { + "0.5": 0.9571126302083334, + "0.6": 0.9734700520833334, + "0.7": 0.979248046875, + "0.8": 0.9825032552083334, + "0.9": 0.9861653645833334 + }, + "dedup_dcs_abs": { + "0.5": 0.7249575877261426, + "0.6": 0.7403489381055504, + "0.7": 0.7451480315357912, + "0.8": 0.7475080401248609, + "0.9": 0.7496708247927018 + }, + "matched_frac_abs": { + "0.5": 0.7779947916666666, + "0.6": 0.744140625, + "0.7": 0.68994140625, + "0.8": 0.5997721354166666, + "0.9": 0.4322916666666667 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.7590634226799011, + "dcs_abs": 0.7603106498718262, + "frac_sign_flipped": 0.020345052083333332, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.7559961080551147, + "0.6": 0.758483350276947, + "0.7": 0.7592421174049377, + "0.8": 0.7593920826911926, + "0.9": 0.7590634226799011 + }, + "unique_frac_signed": { + "0.5": 0.98193359375, + "0.6": 0.9952799479166666, + "0.7": 0.9991861979166666, + "0.8": 0.9995930989583334, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7423379752435721, + "0.6": 0.7549032693592986, + "0.7": 0.7586242445880391, + "0.8": 0.7590830852617122, + "0.9": 0.7590634226799011 + }, + "matched_frac_signed": { + "0.5": 0.7945963541666666, + "0.6": 0.7596028645833334, + "0.7": 0.6985677083333334, + "0.8": 0.6025390625, + "0.9": 0.4405924479166667 + }, + "matched_cos_abs": { + "0.5": 0.7594281435012817, + "0.6": 0.7625158429145813, + "0.7": 0.7629983425140381, + "0.8": 0.7626556754112244, + "0.9": 0.7618380784988403 + }, + "unique_frac_abs": { + "0.5": 0.9569498697916666, + "0.6": 0.9736328125, + "0.7": 0.9794921875, + "0.8": 0.9820149739583334, + "0.9": 0.9860026041666666 + }, + "dedup_dcs_abs": { + "0.5": 0.7267346630396787, + "0.6": 0.742410444712732, + "0.7": 0.7473509155679494, + "0.8": 0.7489392932281287, + "0.9": 0.751174329353186 + }, + "matched_frac_abs": { + "0.5": 0.7801106770833334, + "0.6": 0.7501627604166666, + "0.7": 0.68994140625, + "0.8": 0.5960286458333334, + "0.9": 0.4363606770833333 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 78, + "n_dense_std": 0.5773502691896257, + "l0_mean": 64, + "l0_std": 0.23501035966487338, + "recon_r2_mean": 0.9358003532098967, + "recon_r2_std": 0.013468988454439857, + "l1_per_token_mean": 78.23672614780178, + "l1_per_token_std": 0.21028671581683345, + "babel_mu1_signed_mean": 0.29108617703119916, + "babel_mu1_signed_std": 0.00043631203482011566, + "n_dup_signed_per_threshold": { + "0.5": 212.0, + "0.6": 49.666666666666664, + "0.7": 8.666666666666666, + "0.8": 3.6666666666666665, + "0.9": 0.0 + }, + "n_dup_signed_band": 15.499999999999998, + "n_dup_signed_band_std": 2.6100766272276377, + "n_dup_clusters_signed_per_threshold": { + "0.5": 102.66666666666667, + "0.6": 23.0, + "0.7": 3.3333333333333335, + "0.8": 1.3333333333333333, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 6.916666666666666, + "n_dup_clusters_signed_band_std": 0.7637626158259734, + "largest_dup_signed_per_threshold": { + "0.5": 5.0, + "0.6": 5.0, + "0.7": 3.6666666666666665, + "0.8": 3.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 3.1666666666666665, + "largest_dup_signed_band_std": 0.7637626158259733, + "dcs_signed_mean": 0.7579889496167501, + "dcs_signed_std": 0.0010512944800334333, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7411327407656548, + "0.6": 0.7540988469295876, + "0.7": 0.7573646596881897, + "0.8": 0.7578570853576013, + "0.9": 0.7579889496167501 + }, + "dedup_dcs_signed_band": 0.7568273853980322, + "dedup_dcs_signed_band_std": 0.0010733604561279585, + "matched_cos_signed_per_threshold": { + "0.5": 0.7545605897903442, + "0.6": 0.7573862671852112, + "0.7": 0.7580225865046183, + "0.8": 0.7581449945767721, + "0.9": 0.7579889496167501 + }, + "matched_cos_signed_band": 0.757885699470838, + "matched_cos_signed_band_std": 0.001084465362402629, + "unique_frac_signed_per_threshold": { + "0.5": 0.9822048611111112, + "0.6": 0.9956597222222222, + "0.7": 0.9991319444444443, + "0.8": 0.9996202256944445, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9986029730902778, + "unique_frac_signed_band_std": 0.0001527008768608843, + "matched_frac_signed_per_threshold": { + "0.5": 0.7928059895833334, + "0.6": 0.7587348090277778, + "0.7": 0.6987847222222223, + "0.8": 0.6027018229166666, + "0.9": 0.43842230902777785 + }, + "matched_frac_signed_band": 0.624660915798611, + "matched_frac_signed_band_std": 0.0008242466690667472, + "babel_mu1_abs_mean": 0.3352114260196686, + "babel_mu1_abs_std": 0.0001415059835918215, + "n_dup_abs_per_threshold": { + "0.5": 519.0, + "0.6": 316.6666666666667, + "0.7": 248.66666666666666, + "0.8": 215.0, + "0.9": 170.0 + }, + "n_dup_abs_band": 237.58333333333334, + "n_dup_abs_band_std": 7.571877794400365, + "n_dup_clusters_abs_per_threshold": { + "0.5": 255.0, + "0.6": 156.33333333333334, + "0.7": 123.33333333333333, + "0.8": 107.0, + "0.9": 85.0 + }, + "n_dup_clusters_abs_band": 117.91666666666667, + "n_dup_clusters_abs_band_std": 3.1655699855370965, + "largest_dup_abs_per_threshold": { + "0.5": 5.333333333333333, + "0.6": 5.333333333333333, + "0.7": 3.6666666666666665, + "0.8": 3.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 3.5, + "largest_dup_abs_band_std": 0.9013878188659973, + "dcs_abs_mean": 0.7594329714775085, + "dcs_abs_std": 0.0009107166296511785, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.726080648670581, + "0.6": 0.741569615399284, + "0.7": 0.7463795206907283, + "0.8": 0.7483843225652689, + "0.9": 0.7505427522830561 + }, + "dedup_dcs_abs_band": 0.7467190527345843, + "dedup_dcs_abs_band_std": 0.0009367438291623652, + "matched_cos_abs_per_threshold": { + "0.5": 0.7586802442868551, + "0.6": 0.761440098285675, + "0.7": 0.7619221409161886, + "0.8": 0.7617750366528829, + "0.9": 0.7610719402631124 + }, + "matched_cos_abs_band": 0.7615523040294648, + "matched_cos_abs_band_std": 0.0009420496086039183, + "unique_frac_abs_per_threshold": { + "0.5": 0.95703125, + "0.6": 0.9739040798611112, + "0.7": 0.9796006944444445, + "0.8": 0.982421875, + "0.9": 0.9861653645833334 + }, + "unique_frac_abs_band": 0.9805230034722223, + "unique_frac_abs_band_std": 0.0003595571870066224, + "matched_frac_abs_per_threshold": { + "0.5": 0.7787000868055555, + "0.6": 0.7486436631944443, + "0.7": 0.6908094618055555, + "0.8": 0.5966254340277778, + "0.9": 0.43462456597222227 + }, + "matched_frac_abs_band": 0.61767578125, + "matched_frac_abs_band_std": 0.0009942050555721781, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 6144 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed0/config.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1f7e162da1f7234843065d629bdbb933d9c48c76 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed0/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..f98dcaa42c308340be5f0fda5ad9c697eba20eae --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.12163701171875, + "l0_per_token_0.001": 64.12163701171875, + "l0_per_token_0.01": 64.12163701171875, + "l0_per_token_0.05": 64.12163701171875, + "l0_per_token_0.1": 64.12163701171875, + "l0_per_token_0.5": 57.488536328125, + "l0_per_token_1.0": 21.005958544921874, + "l1_per_token": 78.22586933257212, + "r2_score": 0.9281265595421594, + "l2_loss": 0.047851251459368145 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed1/config.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..28c0933da485f4d5bb12131f345860d2a2df616d --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed1/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..5f1c1741519154e730c4163786debc0188369516 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.46833608398437, + "l0_per_token_0.001": 64.46833608398437, + "l0_per_token_0.01": 64.46833608398437, + "l0_per_token_0.05": 64.46833608398437, + "l0_per_token_0.1": 64.46833608398437, + "l0_per_token_0.5": 57.98992416992188, + "l0_per_token_1.0": 21.01419462890625, + "l1_per_token": 78.45223097013593, + "r2_score": 0.9279219473142333, + "l2_loss": 0.04798747634449653 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed2/config.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4130da590b75251f4fc34dbd2b8480e46e402163 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed2/config.json @@ -0,0 +1,12 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "batchtopk", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 64, + "top_k_inference": 0, + "class_type": "BatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..7a2c6b06df884580df69f27dcb8d4e0cf991c688 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.0201421875, + "l0_per_token_0.001": 64.0201421875, + "l0_per_token_0.01": 64.0201421875, + "l0_per_token_0.05": 64.0201421875, + "l0_per_token_0.1": 64.0201421875, + "l0_per_token_0.5": 57.400218896484375, + "l0_per_token_1.0": 20.8425357421875, + "l1_per_token": 78.03207814069725, + "r2_score": 0.9513525527732973, + "l2_loss": 0.04789464902072545 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/batchtopk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/model.safetensors b/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..17d808082f7df7cbf529d59cf987fd8d2dc827c5 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a5aae09e7eb5a463e8163513cd7b0a9a4dbdb86933eea8d0cd028174a333f21 +size 75550008 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..15fc72308a47ed71de3917671ae6d86e0ecfa0d5 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1d0e143d9ffbaf873485e1df60ad7dbfd0d8c3ca3214bbd4fe8924032938705 +size 99648 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fbab69690747fc9f313b86fe4512324005b4c86 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b20563f9b5732c1c9d0c2f98ee0e8049277380d8a944d833747fe61794631d01 +size 50471 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b011837f13daf3579cd87a069be7d65ab6e5740 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edb7c964151b7f713e78b54dcb04f40524f52dbd65bcb39b36850a9815e5674f +size 50382 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..37d2e56601a74ab12db384ef770ac065d656a939 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27263811f93625ffc32550805887124197991326fb9aa93996331e75274c92f7 +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ae988e02340edd43c88ab520ebfe04de4fc2f0e --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aaf0fb1e693108f250c20ff203ff07ff790d04a47fac32c409a2d6e5ea256b1 +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/model.safetensors b/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3059bf9ad9fc3c7208ed0a49f6d7225c847cf3e8 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aaf42c1f047b642ecdf2aa8ca5b15ba15afaf979858419ab43b732a6b0fae8a +size 75550008 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..15fc72308a47ed71de3917671ae6d86e0ecfa0d5 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1d0e143d9ffbaf873485e1df60ad7dbfd0d8c3ca3214bbd4fe8924032938705 +size 99648 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b0048153c246e11b0aa4f9f5a4ddb859aba0ea0 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bac495eb5dac11423786e7cdaeae21b9d5f412437182b8a66d6201ddc6ba08fd +size 50471 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..55286236fc89f3c4bed8761ccbf0067f01eb8432 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c491aa0504539eac6e4183057a732bfc78b358c61835760c9d39b50f5326e04 +size 50382 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2a837ef50414bf742536ef4d73d33b42969ebad --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cc0939ccc149f74c7de618082cb3fe3d109e8620022590fa333a2500ea86774 +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..58f2c7df61de599472b6618e04591a484f41b964 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0a60f390bb095980d6da1baccd7d45e8e0177e7e62925b065c1f85da3233125 +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/model.safetensors b/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9b6cb7c5e379b6dbfc6a3cca3cf8fbe48e98a60c --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2923625498853cf35a8e64a6f4510b058647da79d6e35cddbe984fec0688d043 +size 75550008 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..15fc72308a47ed71de3917671ae6d86e0ecfa0d5 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1d0e143d9ffbaf873485e1df60ad7dbfd0d8c3ca3214bbd4fe8924032938705 +size 99648 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..26b19a47a79354e8051914f7e3fbc6b5502c6386 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ecb6833567de736363de022f0f8b24a19b4bd42c9c1a2a1484bacc3b4288c3a +size 50471 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..396c7872f33ff281f005790d4b750b95fe5ea124 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98c918bd72eca252df96d783c3e6e003276d16d27f0bc830a9956842c3c29dc0 +size 50382 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..33c6710779b2132bd709f7fae64310c87be3f334 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b26d84dfdf25b568073b50eab2ed07d864d86e3d2300fead71a95cd74a848995 +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0e54271c668dac822d2f9827b1d5aa12323824d --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e504867ccd7e419816186ec9ddbf0d66d3ea8bc0a3af3497284005539b2e414f +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/model.safetensors b/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a8cf0b118867c80ca81c4454f0f7173412e909d1 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca187f675674e5432e3bacf5ef5b9eb8e9afe91502d99857ddd62c74c84d1de6 +size 75550008 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..afbab17d2a7c87272f0ca9958f131e0b739dc5fb --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35aececace33b219d2bd406cb84ddcd74a2bd62111eeca19e185f0abe4b21f70 +size 99648 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d74fe03caa26c018389e464bfd79b85125bd3ce --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f35661b45b273e9043e03acf96105184379fa72ee2c3043805e316b643a6bdf6 +size 50471 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..465526a462036808f108e265d49ba802c33eecdf --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4852877e6f8c954d673b712e6b4f7413661b0f83b519be17a865ec60c331ebd7 +size 50382 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..585fec239b7240e3e383947b84f60eda6220e399 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8951aac2a68d46949fa5f728adcae19efb7eddfc7b238a5e8dd925605a79d8b6 +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c49f395ad8b70c7ab94b007729ca0b0debed7e4 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:457fceba2c88616ded85dbf9d443f180aa453f07499567555036f4f5b8c7a3de +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/model.safetensors b/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..67d9f92f8df239a68f87fe8bd45d03b56cd2c084 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3c1a7367b022974d8e0d1bfecb98aafab146caae19a2d9f2d38987fa0a84ede +size 75550008 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..afbab17d2a7c87272f0ca9958f131e0b739dc5fb --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35aececace33b219d2bd406cb84ddcd74a2bd62111eeca19e185f0abe4b21f70 +size 99648 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d89b3ebad157ea77cecb2510a718fc20d92ffc7 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db5b92523ce03f98f1f2e8e83ed5c688b9111770602a2688eeb85958bc0d494c +size 50471 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f40c438ff408adf507082fe2ad6af0ce9dd97c5b --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9109c892899c3869fb1e21026304dc713e479dcea075fbc32c82090047849d74 +size 50382 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..26a186c6a93e15e96db2468d7b8fcaf5a95b7a4b --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:893bae3a2c825dc1b3ee973133dbaab39f3d0bdcc5699d7e4129e35bf6404caf +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd105b6c42742cbfd9aa5b483c29a3334143ab0c --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fc39aa4c6a23000e3f328da762fc7eff8dbe9c724efb75b353473a441f74991 +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/model.safetensors b/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..90cc5be023be9e916530db5df2eb6e92eb69a7fd --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:621ef040fe8d961a4538e4598c972ed98b6da1c52d27ab2e561a15756df93907 +size 75550008 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..afbab17d2a7c87272f0ca9958f131e0b739dc5fb --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35aececace33b219d2bd406cb84ddcd74a2bd62111eeca19e185f0abe4b21f70 +size 99648 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf7281e9492f0c8d2c7f51266f80d9af3e4e27b8 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d357143b87665baf6ed776a45868fb7fad0320e6ad9fecbfa55d39badd02a67a +size 50471 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..79722ca52ac26a2ffca117cfe97f2712437738c4 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aea120a8f9887e3831a47cf3ee786810848376fc8b1d966d064131e263a95d2 +size 50382 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..321df73d88011b20427f7f680d2adb39bc14c2ae --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca3f621888acb2ce819ede01d7cea401c057820374bf281358ad7e95f697d3ce +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..4bc17c7e5e78ad08e976d0b058a18a6183e01d9e --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb967c8f09731db909ea17b0d6a99683ac84b1b0d0926cd318704dca84fc0e96 +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/model.safetensors b/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d84634171702f786490f12e335aa7762fc28efbf --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cdd26332ad1554b542b4c2bbd078515292a4ad0900e062cf4b54c02093a68b2 +size 75550008 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8de83be649f2f0cc84109b3da72b06a86da913fc --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53a972b5048d9cd93f2f67cbe0716fdc3f9f7eef130ca0bcd46ceccf0b967cde +size 99648 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ab36e4fbfec1b24543f382c2b66867ba1356d31 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e97956041a338f4cc04a2e2bc6d3772504da678588e4a8597127b67bd22d9124 +size 50471 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a3df40a8b3e3c9c795439f3bb63f410d4698981 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:990764a96727799bb4496cbe2dfbf66729ecd36f27e869a43c048b5e106d3a69 +size 50382 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e8af739c61d2ad532f306984d06bd142ab5b3b4 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e2d5783c749e475d29060902840cfe9f081e6dbbfe8ac1dacfdca05ea08df2b +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..28881d0e5d81e7ac1659c797fd0fc93ac892fdab --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29669b7af95d03a04243bc2c622b41780a21ed528d351c02c106fed67e17a2ea +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/model.safetensors b/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c4752af2a6c3fb446a6e0a71b92807e7a1b44163 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaf55e15020ec64ce9d28e8c852a59d0ca5f88685c203b0b3a5a013ed06577e5 +size 75550008 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8de83be649f2f0cc84109b3da72b06a86da913fc --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53a972b5048d9cd93f2f67cbe0716fdc3f9f7eef130ca0bcd46ceccf0b967cde +size 99648 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..376cb7db43157a402242aa90c4f469322c9ff54f --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0faff9deb168af936eba9e964435a847b931ae09d0ada2c5428eb865336b74c0 +size 50471 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..be853f4143d394b3f849f12efcb179f2d5ac9dc7 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83814a690113697b9e270cd11b44e44028b1147340426e959a2630917bcb5810 +size 50382 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d843ce117bd6aa2fbe07a378416372c9d3d432a --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:543fb21cf98462695742f2307436633fa7382e5546a9887367d0ca120e22f142 +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7699a4950bfecbe682cd21e6751e8b308e56b89c --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67d2e374c974ca9815e992f8018cc2a2ccdba0fb82e29b8367f026d2d6b79cd3 +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/model.safetensors b/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..28735b589485cd76e485a0924dc8db22c0bcb149 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f6b66f1e85e17995684617fc9969198038a25ac32bf65a1ebb9c9fbbfbe0e60 +size 75550008 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8de83be649f2f0cc84109b3da72b06a86da913fc --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53a972b5048d9cd93f2f67cbe0716fdc3f9f7eef130ca0bcd46ceccf0b967cde +size 99648 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..b8f20948dfb4922d12b483178465032cd1723d94 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66ac5332bae76053ffdf15b97c3c84b635f550719d71ec0975ee51f0073d34ab +size 50471 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b216d2938b41a47c1124fc36768c3cc2674ccc10 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:108cfc69659cd0e06d44645bf85c04805204f3bb3dd7c9514c583c642a451f39 +size 50382 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0976941e827c296f8710a420c5c34572cd132494 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58a34a3da1960302a7ba625ffc7b7584797d9636b5fc6ac81d53dfcd590c13dd +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f61d5086ccba4c6138359e7d257b27eef12adb24 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a414f5b49f02b1f0b9efee17d5323de5add49ec593938bc7d4ad1c97215002b0 +size 50387 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/model.safetensors b/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dc5dce6ad26e070d9c62170733b840a0b056c336 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d40e20810d8ecd11c5ecde99822cb4eb4368d9b9c40a8cca3cd52286d712e281 +size 18890032 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..25369b4d8167f4401220acd8088693c0808dd520 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f8664b39f84a7961b40b0e3ccd646d478422d3cc05283b1ec8476516297458 +size 25920 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7002e3617f3e99b02103e233f5b8da9f6262239c --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3075726ecc50081aa652f207d8ed24d497b01e73e7039cd17f020a0781043d08 +size 13607 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa487d45e499098f16481a9853d16937fe33428c --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc2783758f561ed4ef4ca3a7a5eb160b767b8d81dda9733fa9acf9aa22ac3261 +size 13518 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..caf8948b099f78bf5ea1e3faf4bf2f81646f269e --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b04c3971eff7b98680932d497d2796c472ed27ee0afe5a952d288ce9607d4f4 +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0120155b5979663b58299cdc904ad081d1585d7 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59ead401a5fb0f063eef7a3cbcf53781bee9b33eb8b8e2fde756d98759ff41ff +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/model.safetensors b/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e0542ddccb314da702c04a54d2a9c238daa45b99 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:297d95c462a3e6b3ad866516046e0bbd5e1ab9aa57171bf427b8c06a08cf6dee +size 18890032 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..25369b4d8167f4401220acd8088693c0808dd520 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f8664b39f84a7961b40b0e3ccd646d478422d3cc05283b1ec8476516297458 +size 25920 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c5c29357fcae48b41bdf28ddb9b7e82a9767674 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:516d58f1f7709ccedfac3ec70e7f18627c1ed14acb7f5521ac70e4de8d240ec6 +size 13607 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..01d4110d2452893ca5e800cac8d2935eb2f03f9d --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8b64414532ab239443d2a64c8cd7d18f61a74a870142f1bf20a6cb881509297 +size 13518 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a34d19fb53aaeca0f7f9d4443b0c13b427e4f913 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31532a750d8ae45422ac0fa9b2109bf102ecd071bf88a5e6206d19aa62325f21 +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..865b7f3bae3906e297e70c9f5314ac0888a152ea --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50ba682b3420152946e9502b0a26c7451efccc0ecdce8f354bc4ce9270a7eefc +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/model.safetensors b/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ff57c929b6bfc2963e1a72951ae46e64d3f4c6f3 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e683a1f3d84335ebeb921bfc588a38c02b68e907698c6d0cd7f4550067f39ca3 +size 18890032 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..25369b4d8167f4401220acd8088693c0808dd520 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f8664b39f84a7961b40b0e3ccd646d478422d3cc05283b1ec8476516297458 +size 25920 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..75c980975b688e5662261072d165e846c8c248e6 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ed520c4fd7cbc0c36da1a059b0732a05ed58f5449f0dca937a5cd3b31a6de61 +size 13607 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4ce634d83529c336662f0775a45855c834c7181 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8b643ccd2d773b9a6fd44f17290edfcc6f89b51f58c030ba9564c83fa571c30 +size 13518 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d76e1d7ae97ec9266ffaea84b309e1cf89e79e16 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9f01ccde17f6f637eefcf87d2fbf2de6c46104cd2149c5ea3e2c8c529dee473 +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..58d234bbcf4323e099cd07a135b52008f7e9e94d --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c17c6019527a8180a6eb7b83973378f5ef74138c3e5395a61a3426cc351b5ada +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/model.safetensors b/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f3189b7bbd49b309516c01456c89eb36a1b80985 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59714c46cc6d70dd7dbdf922d092f5f79503358691f837131b04c424bb91b598 +size 18890032 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..22476d226c364cbdda1c2808c14b7ca570e296dc --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8cf66876f6b155c9ad52c0309e49a57494f55abcd7294eb01e0be14019e09c +size 25920 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fe1a0feb7052bccd61c25846d28045521aebe8f --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc884cee46d817f299a73d2b2135b108b496272b7ed7e71deb946efabca1878f +size 13607 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e09853f79c5729db31d9dde9fa63bfd5c04e5f95 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5e8e9de151009a94a9725f38c5bd11af12f4132ffe50334738952ab16d2b415 +size 13518 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a431dc8eddd497c0351c935ae009fc319ed1750 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07761f8099ba867a708cca7414d38d3c953a3b8410a98caa545f3c317ed77153 +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf6b22f158cfdad9e4258f4f313186e811966bd8 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:778bf9f294dc69981bfc7778d1ab988eb30b8dda2e8839be53374e05cca74127 +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/model.safetensors b/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fa4af0bc2bec921d07ea9d60cd380e6529d779c0 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b6bfded9fe85f7047c199812372abebeefb6e3d7b97b7692667aefb0e54d0ee +size 18890032 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..22476d226c364cbdda1c2808c14b7ca570e296dc --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8cf66876f6b155c9ad52c0309e49a57494f55abcd7294eb01e0be14019e09c +size 25920 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..28b4404264fbb6d62cd100fa0c4a44e37ec1602c --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:479ad7d33479037c3e2ee950c00b6d6055ff627d06e91ae599dc07f2aa973ec4 +size 13607 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..38deea8ab8f752e6b94df9f1ac4cb1067868395f --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fafd29475f1956a1c9d4488eb1ae7fd8ce0c96f92c60412257811f52f1b6b2ce +size 13518 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9550ed4a2e39243682fdc5033e9a72fcad9bf71 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a70afe92cc881c091090e8e79db92e92ecafe101f310b9db482d99c37262bd4e +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..14f513775eb96397bc3d124caa982373fdb98c02 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7298e609602a58e1eed11c03e1d9e47143f492e68c471f65b86a4cd198d6ba59 +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/model.safetensors b/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8456909d7a8a8d6ba80b437a5756786916f76f1a --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0e67e5ba61c73ce368bf9e3dda5e99a5cbcb72f26bac9b69eeb2fc28d7f7a85 +size 18890032 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..22476d226c364cbdda1c2808c14b7ca570e296dc --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8cf66876f6b155c9ad52c0309e49a57494f55abcd7294eb01e0be14019e09c +size 25920 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6f19a796260b97ac23a16eb998d5ca95a7cd751 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74f910cf32058d83050acbf2c400ab18485be9d066cd36c6e728b22839b7478a +size 13607 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a94e4df5dc9001e9b10c4e6beb1cc75ebea73230 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:946bba468165c5fd4229b97f78b4f0bf102593d4adf2b2202c07259692391306 +size 13518 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c3330919204af677462eb060f2bf32d385f42fe --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e8e2d88c065169f1585e4c97f71384909589fbb59728e5342bda99b851bbf9e +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..84f554ac4dd1539e8cd19e80e18e96a0d36cd63b --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5f355ccaa7bed156816ff25b2254d02665fb8a4da5b9873e5757e7ca3a8dbec +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/model.safetensors b/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cd36e502bea93db8182aad4dd6fcc5e8f6fa02bd --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75d52f72e85eea0b4986a24b60d0f687823dcc91933f30969b7a3bddaaa28422 +size 18890032 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..05342aec6090b6546064ba3d08efbd5678ed2326 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d32f23fd81286996eebbb066afd1b434695b0b36f36a328d7242c2e286a4e12 +size 25920 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5c789e3129306dd2c2ddf22507a9dbf9ff86dbd --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da4008d10c2750a5d0b2ab4604727ebde111f028e231e8930a18ef9c212858ae +size 13607 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b314818927938b2603955f18cf012f95c1b22479 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd15aa44f662c75b8ca1883f1e9d7335d391ad37e076100265d15221050d2708 +size 13518 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..946c783f963e4a7c891a137786929c19b79e04cd --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:751c32624940df790e95852d3760693e4c3e30ff2267c4632d7e5de13be35e49 +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3d99c18b528b94cbdc1da9ffa5b2d2de1a7a424 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c49b1dc8c9ef258df03268bb8451a241bc2d31d8b63ea9d93821826459c5ccea +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/model.safetensors b/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fce8dbc47dfcb5d802cb1c6586bfb8067b433e17 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7468dfad16455e202d45aaea72e21ecf44874334c2722ea4da60919c1451f6b5 +size 18890032 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..05342aec6090b6546064ba3d08efbd5678ed2326 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d32f23fd81286996eebbb066afd1b434695b0b36f36a328d7242c2e286a4e12 +size 25920 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8f571fd589b27bb7b4bda0326809565bc974ccd --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dc06c4b18f529fcfb58058676d51d3b8d3ad2b14c198adb8a44310d927fe0e1 +size 13607 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..de3caa60dac9a65daedd784e9617bd5945fd140c --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e2b5c0331e5b1914f26fcf20693884c478c4232a73b9f049a65c6a04b4002aa +size 13518 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0b63dc598cf1ccd57b029a25610b4016fb6a18f --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e664a8e0b9b8826136eba097549098fb44352522fcb36e1cc703ce764ecea7bc +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6b780df62dce87fb24fd5118431099ba550104d --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d0a6713fa01412f9c1244ab8582a1f8e1ab741cff4fe3d5dc3163c36294fa37 +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/model.safetensors b/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..67850fbcf8b3e6bdaec229692f730c018ad4ed2b --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cad810718ba8610e618c8e2c4a704665e18ccc39abb2840a7e5e72079206c22 +size 18890032 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..05342aec6090b6546064ba3d08efbd5678ed2326 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d32f23fd81286996eebbb066afd1b434695b0b36f36a328d7242c2e286a4e12 +size 25920 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..a14e2343fd8b2b5e0bd284e00ebe57bde8bccb79 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4538c29f903be50ad686625ccd910739ba5748fbe0c8f2536260592a8c7a49a6 +size 13607 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a407dff226fde6b0daed087cb6c09d4b71e65942 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a963bfca6e432fe64f584ad19164cdd882f6ff511561e80e0497b40731f4cbcc +size 13518 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b14057f9fe731b45a33da4d26df52d59b44d2669 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b70df4b8f4cefaffc16834e207fb4931af5594ad8fa7a2811a384f74a70fb216 +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf94139590e09c72f19e77ecc6b03f1acaf55c39 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89a12aa6b30cd7384325204fc5c661fb2030962d8fe77036be744554397c6149 +size 13523 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/model.safetensors b/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f1cfb4f819a07e449274f62700e83b614148b6bf --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cc21d9a3561c9ea20784c51eb4668edf856b427a518ab03ad16131c085723c5 +size 37776696 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..da8bb52901fc7e8720225d22c98d91c5121ddac6 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62cb2841c9a88a1c8f93b83e30c4831c7f45af5fff1fed059fc426b953645adf +size 50496 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..73449716d0f97e516059cd49b1c66889e822efcb --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87e6e82cb40b3f439fc8ff230d6e361e0ebb32096ded3ccb7393526c5637aa55 +size 25895 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..26052b716588f9153e82139e3c1265ec864599d7 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b72a63f30cdbd217fbe36d959b748e910a3e7c137299e9f6c33b6a893f3a02d +size 25806 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..948d23ad1230ca6bcd87adba6db3ce5787e19895 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa873956fe13e1ea86402b2f771ef5f07919cab97b193e48c94827c174469c9f +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5bbb4dc138d52eb8a0bd0486895b2bbc7341630b --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16f5d0c58b7ae9c021aa58506e6bbd7dbfa464ce01102548878210c54e7ef648 +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/model.safetensors b/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8509bc1d1a7912760f53d831f74b6ba8cc728e3c --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc097a61c6b39dc7d4449888b7ccffcc3b0edc072a81b5ffe168c1ed4912df15 +size 37776696 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..da8bb52901fc7e8720225d22c98d91c5121ddac6 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62cb2841c9a88a1c8f93b83e30c4831c7f45af5fff1fed059fc426b953645adf +size 50496 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..b883503226bf094f414c4489a8201dc0ade7f63c --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee290e2ec7ec6928602b09150258d7f7f6a497b29a9a54300328bd536d9a47bb +size 25895 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf97ecd0907bb3da305fabba5ed34b0c9deab770 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c9d4199c1960ef2cad745048f7789c3157539af1728c4a431c9b13202d20ae5 +size 25806 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0e767c0322aab90655ab8f6791e023e4fc462ac --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ef77b9ad8920d9feee0d636fa0a5738bf518d8177c86ce573b55f7c4325c605 +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c11fb1e9792f4b184c6977a84af27f5e8bdbaded --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0a38e3aa94c1aacb70d7b5592635959c4ea0ee1f2ffaf05c31209c2138162b9 +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/model.safetensors b/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..32898f45bad232c7b16096e5b70852baa1ea2e69 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b923bceae75d2c69daf30bf78eaa20475ff21da9ed3099a900b3c1fac180ab3 +size 37776696 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..da8bb52901fc7e8720225d22c98d91c5121ddac6 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62cb2841c9a88a1c8f93b83e30c4831c7f45af5fff1fed059fc426b953645adf +size 50496 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..ceda121bd6b91bea3b165f469ad52761b876fba2 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d505067984517e46f4f6c5dd2d1fe9c01999acdd5ab00c89f0f688c26e328e04 +size 25895 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dcbafba2948ecc45833a9e014dfc3342d7380c9e --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfae52c9046860963bd81f6900163407e69a26313c6ef941e749cbe585996c0f +size 25806 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..846f9316a5b7573fa6e30efe54b5de137488b00d --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3353ceed79c8360517e3abbb3cb6362a6af66f8c81f8e330818ad253887cdc1f +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6427a84a937afb29e837973a9448e275a5d8eebd --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b32f72ac54f03a583fe634d5a97f61d086bc710e0c4fa546532fa1ea2d316942 +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/model.safetensors b/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cbe49a1d29622c2a27f97094b4a54f0af80a6c9c --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dacd0efb84d231fb3c9eb319232f3296acdd8b3f5b5f87a766d0ae11d91c42b +size 37776696 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d846a9c18af168b9ab640277af5d8fd537dd811 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd44067a3b156c4c657c2ac2d0d08d2df65ac472c009e7ae3a381d80cb26c91 +size 50496 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..19656c75e42d0fbbf14b2d97c05b8b329b3783d8 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:761cba9dac02b28efcb85ffa10c18bf49e2b743b1b8f2cf7d5115ad9d802092d +size 25895 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef604a1b8f5827c09cb244ca111bdea01e05547e --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3f2745ddec2932264482a5563ccaebbea1a74afafd70a6ac533b75c18113639 +size 25806 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2519511e41e62f2f49e40dce233a9d7ccf1ddd27 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb74d93e3fa3a15477cf9aee886aa77a81ab9c6e101e5b91dce17db965d269fe +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1035ff85fe9a08f0b4ae3ef9cfa22f330577fedb --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38bd3e04a4e542bb0403f7e8b03993a9314b9603e325da35fd22768ddf6ecab7 +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/model.safetensors b/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1644635d9321de69c1ac72c91b0aae879be76cbb --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfed3f1325e8cb3bd0a7b06082b3c3a48cbebbea6d588b3d73d365cd053306e1 +size 37776696 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d846a9c18af168b9ab640277af5d8fd537dd811 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd44067a3b156c4c657c2ac2d0d08d2df65ac472c009e7ae3a381d80cb26c91 +size 50496 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6f061647494e6d8feeee6cfe58d2f84bd7c874e --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b192e0e52d6159c6ab89c83367a9a9bbb4840e87a692af35832efe2202793a9 +size 25895 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8b1a98c4ae073b0aa857dbb00f28e372c81c615 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db721ea3b393a6948bda141ceca6a1e7adf218877b7269cf8912dfd7e76f2189 +size 25806 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef584fa1d3ee1a5218e14dece2cad1b9c94d6ead --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecd30dd07ece0c30e461a8b7be28684308131c7d9ae594b292522c1950165a66 +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..94e566d2a1d2acbc0efbbd879a1cdf8a8baaade2 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b94c58677dfb70e89c66a455d09c19f8e6667031e8d15c7881b98384c193e82 +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/model.safetensors b/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..75672aaa49f19b79cd4f2da4af66c2c0f171f3bc --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:101674e6d16826f013f51dfd0488dcf2c14fdcd47ae23c39b13ddb36c5cb989e +size 37776696 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d846a9c18af168b9ab640277af5d8fd537dd811 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd44067a3b156c4c657c2ac2d0d08d2df65ac472c009e7ae3a381d80cb26c91 +size 50496 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..c026c61882c38c4043bd81e9730c00fdf3016f3f --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34acba420b8898a9c12fdda77af565d0fbf52d687c7cb7e619f79d41955b4659 +size 25895 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..4db7b4b878d3086d198753a223737a51481ab6eb --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b57078d5ab8361f5f322a5a901388dfdfdb55be0ce66582bc506c3082ab7a32f +size 25806 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3d280702cb8bd88ae90e6e44a49e2a54f8404d4 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e39e6b69cdee7bc85355f9dcb944e4170a88e589f02e6339d85d21eb2f6c36f +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d41d4df897bc1b9b99927a9b7cf497c49b3a2b1b --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66048b4842527c63001259062e3c775d30117b7b3ac0f6ced4a650b064fbae37 +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/model.safetensors b/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..853c450f2c6739d3411c7a69e8d81ac368718788 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8763bdb1dd7664cb392794ba1a5485d3f39ca8c5906e7bdc294fae11c4a94627 +size 37776696 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..abcd360e514b4464f4c5e0612bc68c71a618868d --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f384933d03e973cacb959093fb9e9bb2cf7197ece15e4b731d8942192fa0463 +size 50496 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1b8c4aaa7d8adb258729af72797b9f1216633da --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2d4abbf9951235d170f837b394068ace3d7beb54641def01e0e18420af95f63 +size 25895 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..742a05755a9591d522dd12a9c2dd5f97ce1fec41 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80f8ba9908d17bc599c640399eaec432357c843828de9bc82dbb45fb131a254d +size 25806 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..758a97420d46b5d3100158a73bc6da1cf991d7bc --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3738158eacc93c737fa0d4589a9a117ab99e61f66c69eb1135b403870e49f91e +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..de35a6998332ad572fadc0a14dd1c7c502e02482 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba04a421df0db7fefc559a4eb65acd360bdd68772db2ce26144511f21c17a89d +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/model.safetensors b/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..42550df872a11fce393824f17762a1c349479f77 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66d6cc43b1049cf7c5561cb7afe003ca18ce0b7882280bb57c5d46c2ccfcbf9a +size 37776696 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..abcd360e514b4464f4c5e0612bc68c71a618868d --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f384933d03e973cacb959093fb9e9bb2cf7197ece15e4b731d8942192fa0463 +size 50496 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..3df831232e7ed05c785a2f13ea86b40fd5984969 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b22dd5f5c0345e2774a9faf9d20cc206d3190fc6a722b5cbc9dce2a78739df0a +size 25895 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7995c7c411a56d314291cdbcc609d63a56d62d9d --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14e8b5b3b5b31f96be36f46a22a25a48fd6b1ac9444fa65028d40490544fa7ea +size 25806 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ac63ce9c469cac02d8f00f2cafcfb8a155db94e --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79c723e19d6d6974a95c3be160befd3b827c618cc2889c4eed379a2afd67825e +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a8ec5f3bbd0ce726433b3ed5a3dd9848011e188 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0a9ca31277803e800368576d7b3e746d51be6380bff9bf590829ce767e2bc6f +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/model.safetensors b/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8745f59a5d2da5f1fa98df7d57bea11da0fcb7db --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee441f48e48ca7d6755cffc1585a7ae2ef9db2707292820860a0a959b97d5592 +size 37776696 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..abcd360e514b4464f4c5e0612bc68c71a618868d --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f384933d03e973cacb959093fb9e9bb2cf7197ece15e4b731d8942192fa0463 +size 50496 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4eaafc9a59daf827bd456bf52b5f4fb420a9537 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f3f5fb6dabbb714ec82990ed152c6a66a8d24d533e7b26d11ab3ad6a5878159 +size 25895 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..00e5f37399eb55317d27214b9d3beb6807f57182 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce153e208d19b4d71c119a86ba3e377af304529e988814e2d78bb655dc13b93f +size 25806 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1e47fcbdf36a3fdf7e1beb01b65361382b47051 --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a98e40660ebd781a5539dccf57742d4c291ae237f70326c9a4ba0a25b84bf13 +size 25811 diff --git a/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..276c7a7070732e262f1bf40c8e28e6d35903e3aa --- /dev/null +++ b/lm/pythia-160m_layer8/isae/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e753b36135d3c6851c82ce73a7b7c4847010b27f5f165e0dd5df9bce258eaea5 +size 25811 diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/code_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..42d24e68da7b4bfb4e1fd71bad6e175c83d16538 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 128.34414166666667, + "recon_r2": 0.9485683833630062, + "l1_per_token": 124.0184141531441 + }, + "seed1": { + "l0": 128.693835, + "recon_r2": 0.9482102803560943, + "l1_per_token": 124.57384435130254 + }, + "seed2": { + "l0": 128.47938833333333, + "recon_r2": 0.9486899882311027, + "l1_per_token": 124.66204726348346 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.5717664998905547, + "mcc_abs": 0.5718072956238268, + "act_iou": 0.24272674796196378, + "act_iou_n_pairs": 12288, + "input_r2": 0.9551826387161195, + "matched_mcc_signed": { + "0.6": 0.8185821520913493, + "0.7": 0.8554548881821945, + "0.8": 0.887414910115436, + "0.9": 0.9346487473649407 + }, + "matched_mcc_abs": { + "0.6": 0.8204404141458614, + "0.7": 0.8594760920113095, + "0.8": 0.8932139730363899, + "0.9": 0.9408481068345864 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5728182562675908, + "mcc_abs": 0.5728532126292497, + "act_iou": 0.24465776466931208, + "act_iou_n_pairs": 12288, + "input_r2": 0.9564384472454005, + "matched_mcc_signed": { + "0.6": 0.8183574369654192, + "0.7": 0.855745305428435, + "0.8": 0.8861501036203963, + "0.9": 0.9252618528719212 + }, + "matched_mcc_abs": { + "0.6": 0.8207382544506805, + "0.7": 0.8598261630759563, + "0.8": 0.8922019864288043, + "0.9": 0.9330108204272881 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5754473739059739, + "mcc_abs": 0.5754851048045141, + "act_iou": 0.24685605262378518, + "act_iou_n_pairs": 12288, + "input_r2": 0.9561842010161833, + "matched_mcc_signed": { + "0.6": 0.8176048644247529, + "0.7": 0.8546497916117167, + "0.8": 0.8846953253261073, + "0.9": 0.932719810849539 + }, + "matched_mcc_abs": { + "0.6": 0.8200782010847338, + "0.7": 0.8586610559324055, + "0.8": 0.8901369202634324, + "0.9": 0.936402397444519 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 128.50578833333336, + "l0_std": 0.17633512651892375, + "recon_r2_mean": 0.9484895506500677, + "recon_r2_std": 0.0002493809511734876, + "l1_per_token_mean": 124.41810192264336, + "l1_per_token_std": 0.34893792176369964, + "input_r2_mean": 0.9559350956592344, + "input_r2_std": 0.0006639306249143605, + "mcc_signed_mean": 0.5733440433547065, + "mcc_signed_std": 0.001895929171957702, + "mcc_abs_mean": 0.5733818710191968, + "mcc_abs_std": 0.001895040860333372, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8181814844938405, + "0.7": 0.8552833284074488, + "0.8": 0.8860867796873132, + "0.9": 0.930876803695467 + }, + "matched_mcc_signed_band": 0.8726070990710174, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8204189565604253, + "0.7": 0.8593211036732238, + "0.8": 0.8918509599095422, + "0.9": 0.9367537749021313 + }, + "matched_mcc_abs_band": 0.8770861987613306, + "act_iou_mean": 0.24474685508502034, + "act_iou_std": 0.0020660934330671735, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/dict_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..0dac992fb0dd2a4cc6cccddbfe01fcb6bea85903 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 128, + "l0_raw": 128.06962690429688, + "recon_r2": 0.9487432926202075, + "l1_per_token": 123.70847968170604, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 374, + "dense_rate": 0.030436197916666668, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.17451798915863037, + "n_dup_abs": { + "0.5": 1008, + "0.6": 902, + "0.7": 852, + "0.8": 824, + "0.9": 692 + }, + "n_dup_clusters_abs": { + "0.5": 504, + "0.6": 451, + "0.7": 426, + "0.8": 412, + "0.9": 346 + }, + "n_dup_removed_abs": { + "0.5": 504, + "0.6": 451, + "0.7": 426, + "0.8": 412, + "0.9": 346 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.27724379301071167, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 128, + "l0_raw": 128.3863048828125, + "recon_r2": 0.948410901209888, + "l1_per_token": 124.23495010267976, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 375, + "dense_rate": 0.030517578125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 1, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.17489242553710938, + "n_dup_abs": { + "0.5": 1006, + "0.6": 924, + "0.7": 870, + "0.8": 820, + "0.9": 702 + }, + "n_dup_clusters_abs": { + "0.5": 503, + "0.6": 462, + "0.7": 435, + "0.8": 410, + "0.9": 351 + }, + "n_dup_removed_abs": { + "0.5": 503, + "0.6": 462, + "0.7": 435, + "0.8": 410, + "0.9": 351 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2782069742679596, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 128, + "l0_raw": 128.19794194335938, + "recon_r2": 0.9654315643732889, + "l1_per_token": 124.35264534581194, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 366, + "dense_rate": 0.02978515625, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8, + "0.6": 2, + "0.7": 2, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 4, + "0.6": 1, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 4, + "0.6": 1, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.17510248720645905, + "n_dup_abs": { + "0.5": 1027, + "0.6": 922, + "0.7": 880, + "0.8": 832, + "0.9": 726 + }, + "n_dup_clusters_abs": { + "0.5": 513, + "0.6": 461, + "0.7": 440, + "0.8": 416, + "0.9": 363 + }, + "n_dup_removed_abs": { + "0.5": 514, + "0.6": 461, + "0.7": 440, + "0.8": 416, + "0.9": 363 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.27692368626594543, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.532895028591156, + "dcs_abs": 0.5365374684333801, + "frac_sign_flipped": 0.07674153645833333, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.5329238176345825, + "0.6": 0.532929539680481, + "0.7": 0.5328950881958008, + "0.8": 0.5328950881958008, + "0.9": 0.5328950881958008 + }, + "unique_frac_signed": { + "0.5": 0.9999186197916666, + "0.6": 0.9999593098958334, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5328804481832776, + "0.6": 0.5329078547219979, + "0.7": 0.5328950881958008, + "0.8": 0.5328950881958008, + "0.9": 0.5328950881958008 + }, + "matched_frac_signed": { + "0.5": 0.588134765625, + "0.6": 0.4568684895833333, + "0.7": 0.2967122395833333, + "0.8": 0.12736002604166666, + "0.9": 0.0302734375 + }, + "matched_cos_abs": { + "0.5": 0.5375755429267883, + "0.6": 0.5376508831977844, + "0.7": 0.5376463532447815, + "0.8": 0.5371770262718201, + "0.9": 0.5366780757904053 + }, + "unique_frac_abs": { + "0.5": 0.9590250651041666, + "0.6": 0.9628499348958334, + "0.7": 0.9649658203125, + "0.8": 0.966552734375, + "0.9": 0.9716389973958334 + }, + "dedup_dcs_abs": { + "0.5": 0.5155484200537709, + "0.6": 0.5176771178836741, + "0.7": 0.5188103542968747, + "0.8": 0.5192099235864589, + "0.9": 0.5214573474853145 + }, + "matched_frac_abs": { + "0.5": 0.5718587239583334, + "0.6": 0.4440104166666667, + "0.7": 0.287109375, + "0.8": 0.12093098958333333, + "0.9": 0.02783203125 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.5336127281188965, + "dcs_abs": 0.5372638702392578, + "frac_sign_flipped": 0.079345703125, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.5336893200874329, + "0.6": 0.533642590045929, + "0.7": 0.5336441397666931, + "0.8": 0.5336127281188965, + "0.9": 0.5336127281188965 + }, + "unique_frac_signed": { + "0.5": 0.9997965494791666, + "0.6": 0.9999186197916666, + "0.7": 0.9999593098958334, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5335807407172979, + "0.6": 0.5335991621007754, + "0.7": 0.5336224257310581, + "0.8": 0.5336127281188965, + "0.9": 0.5336127281188965 + }, + "matched_frac_signed": { + "0.5": 0.5913899739583334, + "0.6": 0.4542643229166667, + "0.7": 0.294189453125, + "0.8": 0.13142903645833334, + "0.9": 0.033203125 + }, + "matched_cos_abs": { + "0.5": 0.5378563404083252, + "0.6": 0.5379647016525269, + "0.7": 0.5379307270050049, + "0.8": 0.5373501181602478, + "0.9": 0.5369553565979004 + }, + "unique_frac_abs": { + "0.5": 0.9585774739583334, + "0.6": 0.962890625, + "0.7": 0.9647623697916666, + "0.8": 0.96630859375, + "0.9": 0.9711507161458334 + }, + "dedup_dcs_abs": { + "0.5": 0.5155769721410859, + "0.6": 0.5180011678021401, + "0.7": 0.5189753229691025, + "0.8": 0.5192460370308254, + "0.9": 0.5214645790983923 + }, + "matched_frac_abs": { + "0.5": 0.5721028645833334, + "0.6": 0.4403483072916667, + "0.7": 0.2841796875, + "0.8": 0.123779296875, + "0.9": 0.029947916666666668 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.5389622449874878, + "dcs_abs": 0.5424779057502747, + "frac_sign_flipped": 0.07771809895833333, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.5390437245368958, + "0.6": 0.5389988422393799, + "0.7": 0.5389988422393799, + "0.8": 0.5389622449874878, + "0.9": 0.5389622449874878 + }, + "unique_frac_signed": { + "0.5": 0.9997965494791666, + "0.6": 0.9999593098958334, + "0.7": 0.9999593098958334, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5389340558103868, + "0.6": 0.5389769103203434, + "0.7": 0.5389769103203434, + "0.8": 0.5389622449874878, + "0.9": 0.5389622449874878 + }, + "matched_frac_signed": { + "0.5": 0.5966796875, + "0.6": 0.4654134114583333, + "0.7": 0.3113606770833333, + "0.8": 0.138427734375, + "0.9": 0.033040364583333336 + }, + "matched_cos_abs": { + "0.5": 0.5427398085594177, + "0.6": 0.5425488352775574, + "0.7": 0.5427301526069641, + "0.8": 0.542367160320282, + "0.9": 0.5419221520423889 + }, + "unique_frac_abs": { + "0.5": 0.9586181640625, + "0.6": 0.9624430338541666, + "0.7": 0.9643961588541666, + "0.8": 0.9663899739583334, + "0.9": 0.970947265625 + }, + "dedup_dcs_abs": { + "0.5": 0.5202802388448617, + "0.6": 0.5221723470385768, + "0.7": 0.5234068744684919, + "0.8": 0.5241381859377725, + "0.9": 0.526177831707173 + }, + "matched_frac_abs": { + "0.5": 0.5790201822916666, + "0.6": 0.4501953125, + "0.7": 0.300048828125, + "0.8": 0.13037109375, + "0.9": 0.030436197916666668 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 372, + "n_dense_std": 4.932882862316248, + "l0_mean": 128, + "l0_std": 0.159285010952997, + "recon_r2_mean": 0.9541952527344616, + "recon_r2_std": 0.009732350458447855, + "l1_per_token_mean": 124.0986917100659, + "l1_per_token_std": 0.3430191144404794, + "babel_mu1_signed_mean": 0.1748376339673996, + "babel_mu1_signed_std": 0.0002960761372727105, + "n_dup_signed_per_threshold": { + "0.5": 4.0, + "0.6": 1.3333333333333333, + "0.7": 0.6666666666666666, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.5, + "n_dup_signed_band_std": 0.5, + "n_dup_clusters_signed_per_threshold": { + "0.5": 2.0, + "0.6": 0.6666666666666666, + "0.7": 0.3333333333333333, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.25, + "n_dup_clusters_signed_band_std": 0.25, + "largest_dup_signed_per_threshold": { + "0.5": 2.0, + "0.6": 1.6666666666666667, + "0.7": 1.3333333333333333, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.25, + "largest_dup_signed_band_std": 0.25, + "dcs_signed_mean": 0.5351566672325134, + "dcs_signed_std": 0.003315205829190176, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5351317482369874, + "0.6": 0.5351613090477055, + "0.7": 0.5351648080824009, + "0.8": 0.5351566871007284, + "0.9": 0.5351566871007284 + }, + "dedup_dcs_signed_band": 0.5351598728328908, + "dedup_dcs_signed_band_std": 0.0033185315984100862, + "matched_cos_signed_per_threshold": { + "0.5": 0.5352189540863037, + "0.6": 0.5351903239885966, + "0.7": 0.5351793567339579, + "0.8": 0.5351566871007284, + "0.9": 0.5351566871007284 + }, + "matched_cos_signed_band": 0.5351707637310028, + "matched_cos_signed_band_std": 0.0033191845603377404, + "unique_frac_signed_per_threshold": { + "0.5": 0.9998372395833334, + "0.6": 0.9999457465277778, + "0.7": 0.999972873263889, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9999796549479167, + "unique_frac_signed_band_std": 1.017252604168517e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.5920681423611112, + "0.6": 0.4588487413194444, + "0.7": 0.30075412326388884, + "0.8": 0.13240559895833334, + "0.9": 0.03217230902777778 + }, + "matched_frac_signed_band": 0.2310451931423611, + "matched_frac_signed_band_std": 0.005214700515648969, + "babel_mu1_abs_mean": 0.2774581511815389, + "babel_mu1_abs_std": 0.0006679589007406599, + "n_dup_abs_per_threshold": { + "0.5": 1013.6666666666666, + "0.6": 916.0, + "0.7": 867.3333333333334, + "0.8": 825.3333333333334, + "0.9": 706.6666666666666 + }, + "n_dup_abs_band": 828.8333333333334, + "n_dup_abs_band_std": 11.250925887825115, + "n_dup_clusters_abs_per_threshold": { + "0.5": 506.6666666666667, + "0.6": 458.0, + "0.7": 433.6666666666667, + "0.8": 412.6666666666667, + "0.9": 353.3333333333333 + }, + "n_dup_clusters_abs_band": 414.4166666666667, + "n_dup_clusters_abs_band_std": 5.625462943912558, + "largest_dup_abs_per_threshold": { + "0.5": 2.3333333333333335, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.0, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.5387597481409708, + "dcs_abs_std": 0.003240437763885207, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5171352103465728, + "0.6": 0.5192835442414637, + "0.7": 0.5203975172448231, + "0.8": 0.5208647155183522, + "0.9": 0.5230332527636267 + }, + "dedup_dcs_abs_band": 0.5208947574420665, + "dedup_dcs_abs_band_std": 0.0026673677679469976, + "matched_cos_abs_per_threshold": { + "0.5": 0.5393905639648438, + "0.6": 0.5393881400426229, + "0.7": 0.5394357442855835, + "0.8": 0.5389647682507833, + "0.9": 0.5385185281435648 + }, + "matched_cos_abs_band": 0.5390767951806387, + "matched_cos_abs_band_std": 0.0028741068218897945, + "unique_frac_abs_per_threshold": { + "0.5": 0.958740234375, + "0.6": 0.9627278645833334, + "0.7": 0.9647081163194443, + "0.8": 0.9664171006944445, + "0.9": 0.9712456597222223 + }, + "unique_frac_abs_band": 0.9662746853298612, + "unique_frac_abs_band_std": 0.00022890067317358066, + "matched_frac_abs_per_threshold": { + "0.5": 0.5743272569444445, + "0.6": 0.44485134548611116, + "0.7": 0.2904459635416667, + "0.8": 0.1250271267361111, + "0.9": 0.029405381944444448 + }, + "matched_frac_abs_band": 0.22243245442708337, + "matched_frac_abs_band_std": 0.004620746085326717, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 12288 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed0/config.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e3fd1b5739d61d18c1705071127e390e70d7b681 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed0/lock.lock b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed0/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..2510e06b181a8b82620dd7d9fd8f23c0cd759625 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.06962690429688, + "l0_per_token_0.001": 128.06962690429688, + "l0_per_token_0.01": 128.06962690429688, + "l0_per_token_0.05": 128.06962690429688, + "l0_per_token_0.1": 128.06962690429688, + "l0_per_token_0.5": 92.38570747070312, + "l0_per_token_1.0": 30.087119189453126, + "l1_per_token": 123.70847968170604, + "r2_score": 0.9487432926202075, + "l2_loss": 0.034125228710157544 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed1/config.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..55945e3b61518cde4bc2bb08acdd35c4631f8042 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed1/lock.lock b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed1/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..607eb8816382f98969ff1b523fe9b8fa551dc5ba --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.3863048828125, + "l0_per_token_0.001": 128.3863048828125, + "l0_per_token_0.01": 128.3863048828125, + "l0_per_token_0.05": 128.3863048828125, + "l0_per_token_0.1": 128.3863048828125, + "l0_per_token_0.5": 93.15153515625, + "l0_per_token_1.0": 30.32521826171875, + "l1_per_token": 124.23495010267976, + "r2_score": 0.948410901209888, + "l2_loss": 0.03434652526778455 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed2/config.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1689a729a9e579f9e7806492669b4a718217b682 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed2/lock.lock b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed2/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..dc05c34f6d1b28019460afae167e7a69021f310d --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.19794194335938, + "l0_per_token_0.001": 128.19794194335938, + "l0_per_token_0.01": 128.19794194335938, + "l0_per_token_0.05": 128.19794194335938, + "l0_per_token_0.1": 128.19794194335938, + "l0_per_token_0.5": 92.71353432617188, + "l0_per_token_1.0": 30.23004580078125, + "l1_per_token": 124.35264534581194, + "r2_score": 0.9654315643732889, + "l2_loss": 0.03403350403611902 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/code_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..89bc5e7ce959bb77b9b9290f44ede562d944a85f --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 32.11302833333333, + "recon_r2": 0.9049568411215114, + "l1_per_token": 63.9235978486073 + }, + "seed1": { + "l0": 32.22569166666667, + "recon_r2": 0.9048404488409074, + "l1_per_token": 64.11336626311382 + }, + "seed2": { + "l0": 32.280586666666665, + "recon_r2": 0.9043707849050284, + "l1_per_token": 64.18761559074322 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.522133541288584, + "mcc_abs": 0.5221652309135364, + "act_iou": 0.2597599619487409, + "act_iou_n_pairs": 12288, + "input_r2": 0.9575777298004101, + "matched_mcc_signed": { + "0.6": 0.800364658951539, + "0.7": 0.8353041065710233, + "0.8": 0.8720917217041179, + "0.9": 0.9141696595587123 + }, + "matched_mcc_abs": { + "0.6": 0.7961708977494727, + "0.7": 0.8334828459066819, + "0.8": 0.8709983612942828, + "0.9": 0.9138592931400333 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.521046609544308, + "mcc_abs": 0.5210775681351393, + "act_iou": 0.2612771363917162, + "act_iou_n_pairs": 12288, + "input_r2": 0.9574929236454984, + "matched_mcc_signed": { + "0.6": 0.8016281559462827, + "0.7": 0.8368759399834665, + "0.8": 0.874743758429976, + "0.9": 0.9134434356624537 + }, + "matched_mcc_abs": { + "0.6": 0.7964625240726925, + "0.7": 0.8348055558116831, + "0.8": 0.8734750278930443, + "0.9": 0.9126719820256713 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5203742015902033, + "mcc_abs": 0.5204061423448976, + "act_iou": 0.25935175697914115, + "act_iou_n_pairs": 12288, + "input_r2": 0.957076597567088, + "matched_mcc_signed": { + "0.6": 0.8008476670490481, + "0.7": 0.8374932023519629, + "0.8": 0.8752770014209511, + "0.9": 0.9176719548273047 + }, + "matched_mcc_abs": { + "0.6": 0.7976138222281608, + "0.7": 0.8353808483006083, + "0.8": 0.8744273590264876, + "0.9": 0.9174782914396971 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 32.20643555555555, + "l0_std": 0.08542275533312943, + "recon_r2_mean": 0.9047226916224824, + "recon_r2_std": 0.0003102668917045233, + "l1_per_token_mean": 64.07485990082144, + "l1_per_token_std": 0.1361557820784001, + "input_r2_mean": 0.9573824170043322, + "input_r2_std": 0.00026822037045123734, + "mcc_signed_mean": 0.5211847841410318, + "mcc_signed_std": 0.0008877714840877767, + "mcc_abs_mean": 0.5212163137978577, + "mcc_abs_std": 0.0008877138712847375, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8009468273156233, + "0.7": 0.8365577496354842, + "0.8": 0.8740374938516816, + "0.9": 0.9150950166828236 + }, + "matched_mcc_signed_band": 0.8566592718714032, + "matched_mcc_abs_per_threshold": { + "0.6": 0.7967490813501087, + "0.7": 0.8345564166729912, + "0.8": 0.8729669160712715, + "0.9": 0.9146698555351339 + }, + "matched_mcc_abs_band": 0.8547355674073763, + "act_iou_mean": 0.2601296184398661, + "act_iou_std": 0.0010145225042674892, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/dict_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..07cffa56842972ee7895212953efd0a82859a98b --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 32.01470703125, + "recon_r2": 0.9053322978451912, + "l1_per_token": 63.77330868725877, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 21, + "dense_rate": 0.001708984375, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 31, + "0.6": 7, + "0.7": 3, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 15, + "0.6": 3, + "0.7": 1, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 16, + "0.6": 4, + "0.7": 2, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.21547822654247284, + "n_dup_abs": { + "0.5": 351, + "0.6": 189, + "0.7": 119, + "0.8": 74, + "0.9": 38 + }, + "n_dup_clusters_abs": { + "0.5": 175, + "0.6": 94, + "0.7": 59, + "0.8": 37, + "0.9": 19 + }, + "n_dup_removed_abs": { + "0.5": 176, + "0.6": 95, + "0.7": 60, + "0.8": 37, + "0.9": 19 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.26000797748565674, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 32.10383325195313, + "recon_r2": 0.9051993471737279, + "l1_per_token": 63.91419169174386, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 23, + "dense_rate": 0.0018717447916666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 55, + "0.6": 15, + "0.7": 3, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 27, + "0.6": 7, + "0.7": 1, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 28, + "0.6": 8, + "0.7": 2, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.21601593494415283, + "n_dup_abs": { + "0.5": 337, + "0.6": 179, + "0.7": 115, + "0.8": 68, + "0.9": 40 + }, + "n_dup_clusters_abs": { + "0.5": 168, + "0.6": 89, + "0.7": 57, + "0.8": 34, + "0.9": 20 + }, + "n_dup_removed_abs": { + "0.5": 169, + "0.6": 90, + "0.7": 58, + "0.8": 34, + "0.9": 20 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 3, + "0.7": 3, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2593936026096344, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 32, + "l0_raw": 32.196017236328125, + "recon_r2": 0.9356035650656311, + "l1_per_token": 64.03914689621014, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 22, + "dense_rate": 0.0017903645833333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 50, + "0.6": 20, + "0.7": 8, + "0.8": 3, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 24, + "0.6": 9, + "0.7": 3, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 26, + "0.6": 11, + "0.7": 5, + "0.8": 2, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 4, + "0.6": 4, + "0.7": 4, + "0.8": 3, + "0.9": 1 + }, + "babel_mu1_signed": 0.21611519157886505, + "n_dup_abs": { + "0.5": 321, + "0.6": 183, + "0.7": 115, + "0.8": 75, + "0.9": 38 + }, + "n_dup_clusters_abs": { + "0.5": 159, + "0.6": 90, + "0.7": 56, + "0.8": 37, + "0.9": 19 + }, + "n_dup_removed_abs": { + "0.5": 162, + "0.6": 93, + "0.7": 59, + "0.8": 38, + "0.9": 19 + }, + "largest_dup_abs": { + "0.5": 4, + "0.6": 4, + "0.7": 4, + "0.8": 3, + "0.9": 2 + }, + "babel_mu1_abs": 0.2592376172542572, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.5773086547851562, + "dcs_abs": 0.5817011594772339, + "frac_sign_flipped": 0.10213216145833333, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.5770264863967896, + "0.6": 0.5772801041603088, + "0.7": 0.5772498250007629, + "0.8": 0.5772793889045715, + "0.9": 0.5773086547851562 + }, + "unique_frac_signed": { + "0.5": 0.9982096354166666, + "0.6": 0.99951171875, + "0.7": 0.9998372395833334, + "0.8": 0.9999186197916666, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5759933986118995, + "0.6": 0.5769982291094493, + "0.7": 0.577155871578725, + "0.8": 0.5772324097876359, + "0.9": 0.5773086547851562 + }, + "matched_frac_signed": { + "0.5": 0.61083984375, + "0.6": 0.539794921875, + "0.7": 0.4407552083333333, + "0.8": 0.3077799479166667, + "0.9": 0.13240559895833334 + }, + "matched_cos_abs": { + "0.5": 0.5804751515388489, + "0.6": 0.5808290243148804, + "0.7": 0.5810995101928711, + "0.8": 0.5812856554985046, + "0.9": 0.5813443064689636 + }, + "unique_frac_abs": { + "0.5": 0.9859619140625, + "0.6": 0.9924723307291666, + "0.7": 0.9951985677083334, + "0.8": 0.9971110026041666, + "0.9": 0.9984130859375 + }, + "dedup_dcs_abs": { + "0.5": 0.5723263914769632, + "0.6": 0.5764567355169371, + "0.7": 0.5783094002399594, + "0.8": 0.5796063227535342, + "0.9": 0.5804217630138737 + }, + "matched_frac_abs": { + "0.5": 0.6060384114583334, + "0.6": 0.5381673177083334, + "0.7": 0.43994140625, + "0.8": 0.3070475260416667, + "0.9": 0.1318359375 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.5780982375144958, + "dcs_abs": 0.5826689600944519, + "frac_sign_flipped": 0.10611979166666667, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.5779822468757629, + "0.6": 0.5781297087669373, + "0.7": 0.5781452655792236, + "0.8": 0.5781002640724182, + "0.9": 0.5780982375144958 + }, + "unique_frac_signed": { + "0.5": 0.998291015625, + "0.6": 0.9993896484375, + "0.7": 0.9997151692708334, + "0.8": 0.9998779296875, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5769944842468249, + "0.6": 0.5777768463958637, + "0.7": 0.5779805920416645, + "0.8": 0.5780296951925266, + "0.9": 0.5780982375144958 + }, + "matched_frac_signed": { + "0.5": 0.607666015625, + "0.6": 0.53857421875, + "0.7": 0.444580078125, + "0.8": 0.3153483072916667, + "0.9": 0.14013671875 + }, + "matched_cos_abs": { + "0.5": 0.5815528631210327, + "0.6": 0.5819079279899597, + "0.7": 0.5822558999061584, + "0.8": 0.5822622776031494, + "0.9": 0.5824219584465027 + }, + "unique_frac_abs": { + "0.5": 0.9862467447916666, + "0.6": 0.9923502604166666, + "0.7": 0.9951578776041666, + "0.8": 0.9969482421875, + "0.9": 0.9984537760416666 + }, + "dedup_dcs_abs": { + "0.5": 0.5735546181773922, + "0.6": 0.5774564838793594, + "0.7": 0.5794365455731167, + "0.8": 0.58048535414855, + "0.9": 0.5815214036604932 + }, + "matched_frac_abs": { + "0.5": 0.6031087239583334, + "0.6": 0.5387369791666666, + "0.7": 0.4436848958333333, + "0.8": 0.314453125, + "0.9": 0.13972981770833334 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.5779923796653748, + "dcs_abs": 0.5820271968841553, + "frac_sign_flipped": 0.10416666666666667, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.5773341655731201, + "0.6": 0.5779677033424377, + "0.7": 0.5780386328697205, + "0.8": 0.5779980421066284, + "0.9": 0.5779924392700195 + }, + "unique_frac_signed": { + "0.5": 0.997802734375, + "0.6": 0.9992268880208334, + "0.7": 0.9997151692708334, + "0.8": 0.9998779296875, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5760656090569682, + "0.6": 0.5775208695874123, + "0.7": 0.5778739897044337, + "0.8": 0.5779274857050041, + "0.9": 0.5779924392700195 + }, + "matched_frac_signed": { + "0.5": 0.6089680989583334, + "0.6": 0.5372721354166666, + "0.7": 0.4413248697916667, + "0.8": 0.3138020833333333, + "0.9": 0.14119466145833334 + }, + "matched_cos_abs": { + "0.5": 0.5808130502700806, + "0.6": 0.5816501975059509, + "0.7": 0.5816540122032166, + "0.8": 0.581687331199646, + "0.9": 0.5818431973457336 + }, + "unique_frac_abs": { + "0.5": 0.9865315755208334, + "0.6": 0.9925537109375, + "0.7": 0.9952392578125, + "0.8": 0.9970703125, + "0.9": 0.9984130859375 + }, + "dedup_dcs_abs": { + "0.5": 0.5729904135660036, + "0.6": 0.5773190620020614, + "0.7": 0.5788849074087921, + "0.8": 0.579983169096522, + "0.9": 0.5809198621936957 + }, + "matched_frac_abs": { + "0.5": 0.6035970052083334, + "0.6": 0.5359700520833334, + "0.7": 0.44091796875, + "0.8": 0.31298828125, + "0.9": 0.14054361979166666 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 22, + "n_dense_std": 1.0, + "l0_mean": 32, + "l0_std": 0.09065939982295004, + "recon_r2_mean": 0.9153784033615168, + "recon_r2_std": 0.017515629975494534, + "l1_per_token_mean": 63.90888242507092, + "l1_per_token_std": 0.13299860739523411, + "babel_mu1_signed_mean": 0.21586978435516357, + "babel_mu1_signed_std": 0.0003427114097793699, + "n_dup_signed_per_threshold": { + "0.5": 45.333333333333336, + "0.6": 14.0, + "0.7": 4.666666666666667, + "0.8": 2.3333333333333335, + "0.9": 0.0 + }, + "n_dup_signed_band": 5.25, + "n_dup_signed_band_std": 2.384848003542364, + "n_dup_clusters_signed_per_threshold": { + "0.5": 22.0, + "0.6": 6.333333333333333, + "0.7": 1.6666666666666667, + "0.8": 1.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 2.25, + "n_dup_clusters_signed_band_std": 1.0, + "largest_dup_signed_per_threshold": { + "0.5": 3.3333333333333335, + "0.6": 3.3333333333333335, + "0.7": 3.3333333333333335, + "0.8": 2.3333333333333335, + "0.9": 1.0 + }, + "largest_dup_signed_band": 2.5, + "largest_dup_signed_band_std": 0.4330127018922193, + "dcs_signed_mean": 0.5777997573216757, + "dcs_signed_std": 0.00042858808556346304, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5763511639718976, + "0.6": 0.5774319816975751, + "0.7": 0.5776701511082744, + "0.8": 0.5777298635617222, + "0.9": 0.5777997771898905 + }, + "dedup_dcs_signed_band": 0.5776579433893655, + "dedup_dcs_signed_band_std": 0.0004253109977407672, + "matched_cos_signed_per_threshold": { + "0.5": 0.5774476329485575, + "0.6": 0.5777925054232279, + "0.7": 0.5778112411499023, + "0.8": 0.5777925650278727, + "0.9": 0.5777997771898905 + }, + "matched_cos_signed_band": 0.5777990221977233, + "matched_cos_signed_band_std": 0.0004538533070858928, + "unique_frac_signed_per_threshold": { + "0.5": 0.9981011284722222, + "0.6": 0.9993760850694445, + "0.7": 0.999755859375, + "0.8": 0.9998914930555555, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.999755859375, + "unique_frac_signed_band_std": 5.6638227974715456e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.6091579861111112, + "0.6": 0.5385470920138888, + "0.7": 0.4422200520833333, + "0.8": 0.31231011284722227, + "0.9": 0.13791232638888892 + }, + "matched_frac_signed_band": 0.3577473958333333, + "matched_frac_signed_band_std": 0.002307885890704312, + "babel_mu1_abs_mean": 0.2595463991165161, + "babel_mu1_abs_std": 0.00040727607456972836, + "n_dup_abs_per_threshold": { + "0.5": 336.3333333333333, + "0.6": 183.66666666666666, + "0.7": 116.33333333333333, + "0.8": 72.33333333333333, + "0.9": 38.666666666666664 + }, + "n_dup_abs_band": 102.75, + "n_dup_abs_band_std": 2.25, + "n_dup_clusters_abs_per_threshold": { + "0.5": 167.33333333333334, + "0.6": 91.0, + "0.7": 57.333333333333336, + "0.8": 36.0, + "0.9": 19.333333333333332 + }, + "n_dup_clusters_abs_band": 50.91666666666667, + "n_dup_clusters_abs_band_std": 1.181453906563152, + "largest_dup_abs_per_threshold": { + "0.5": 3.3333333333333335, + "0.6": 3.3333333333333335, + "0.7": 3.3333333333333335, + "0.8": 2.3333333333333335, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.75, + "largest_dup_abs_band_std": 0.4330127018922193, + "dcs_abs_mean": 0.5821324388186137, + "dcs_abs_std": 0.000492408780636887, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.572957141073453, + "0.6": 0.577077427132786, + "0.7": 0.5788769510739561, + "0.8": 0.580024948666202, + "0.9": 0.5809543429560209 + }, + "dedup_dcs_abs_band": 0.5792334174572413, + "dedup_dcs_abs_band_std": 0.0005145659699607918, + "matched_cos_abs_per_threshold": { + "0.5": 0.5809470216433207, + "0.6": 0.5814623832702637, + "0.7": 0.581669807434082, + "0.8": 0.5817450881004333, + "0.9": 0.5818698207537333 + }, + "matched_cos_abs_band": 0.581686774889628, + "matched_cos_abs_band_std": 0.0005365315506148809, + "unique_frac_abs_per_threshold": { + "0.5": 0.9862467447916666, + "0.6": 0.992458767361111, + "0.7": 0.9951985677083334, + "0.8": 0.9970431857638888, + "0.9": 0.9984266493055555 + }, + "unique_frac_abs_band": 0.9957817925347221, + "unique_frac_abs_band_std": 4.8073482526226046e-05, + "matched_frac_abs_per_threshold": { + "0.5": 0.604248046875, + "0.6": 0.5376247829861112, + "0.7": 0.4415147569444444, + "0.8": 0.3114963107638889, + "0.9": 0.13736979166666666 + }, + "matched_frac_abs_band": 0.3570014105902778, + "matched_frac_abs_band_std": 0.002506683255576416, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 12288 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed0/config.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..67cc265799484f523eb2550325684a58cd814b2e --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed0/lock.lock b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed0/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..2f8ee9b10b5d3911a643112b68299ee90805c2dc --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.01470703125, + "l0_per_token_0.001": 32.01470703125, + "l0_per_token_0.01": 32.01470703125, + "l0_per_token_0.05": 32.01470703125, + "l0_per_token_0.1": 32.01470703125, + "l0_per_token_0.5": 32.01470703125, + "l0_per_token_1.0": 22.1821138671875, + "l1_per_token": 63.77330868725877, + "r2_score": 0.9053322978451912, + "l2_loss": 0.06302700958843767 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed1/config.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec4bf09a23cb93d54f4886a0d5a1358212b2dd3b --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed1/lock.lock b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed1/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..062dd350f41f4711329f8b866767887fe078a0fb --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.10383325195313, + "l0_per_token_0.001": 32.10383325195313, + "l0_per_token_0.01": 32.10383325195313, + "l0_per_token_0.05": 32.10383325195313, + "l0_per_token_0.1": 32.10383325195313, + "l0_per_token_0.5": 32.10383325195313, + "l0_per_token_1.0": 22.23313330078125, + "l1_per_token": 63.91419169174386, + "r2_score": 0.9051993471737279, + "l2_loss": 0.06311552428832348 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed2/config.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c1789120be181f0a18e633a58926446a9bc1d5a1 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed2/lock.lock b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed2/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..c9ec43f4a22c2d71262d51017ad2ba4f70306e04 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.196017236328125, + "l0_per_token_0.001": 32.196017236328125, + "l0_per_token_0.01": 32.196017236328125, + "l0_per_token_0.05": 32.196017236328125, + "l0_per_token_0.1": 32.196017236328125, + "l0_per_token_0.5": 32.196017236328125, + "l0_per_token_1.0": 22.280626416015625, + "l1_per_token": 64.03914689621014, + "r2_score": 0.9356035650656311, + "l2_loss": 0.06339992795499932 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/code_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..c331065206508dd02b9f66d06ba504195dd053f3 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 64.22754833333333, + "recon_r2": 0.9274573039318837, + "l1_per_token": 86.39629357177725 + }, + "seed1": { + "l0": 64.63336833333334, + "recon_r2": 0.9271365248638103, + "l1_per_token": 86.7798685908259 + }, + "seed2": { + "l0": 64.26007, + "recon_r2": 0.9275269656796971, + "l1_per_token": 86.21737778629452 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.5871430128471592, + "mcc_abs": 0.5871719348259906, + "act_iou": 0.2551322878625298, + "act_iou_n_pairs": 12288, + "input_r2": 0.9564838134171233, + "matched_mcc_signed": { + "0.6": 0.8025695116240891, + "0.7": 0.8403461290293527, + "0.8": 0.8792938967299654, + "0.9": 0.9256497196535318 + }, + "matched_mcc_abs": { + "0.6": 0.7993168228673645, + "0.7": 0.8397588149397954, + "0.8": 0.8796583429143116, + "0.9": 0.9263403427459719 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5869455069387766, + "mcc_abs": 0.5869769107742878, + "act_iou": 0.25567944129999165, + "act_iou_n_pairs": 12288, + "input_r2": 0.9575274862649419, + "matched_mcc_signed": { + "0.6": 0.8041154188079526, + "0.7": 0.8426465658183735, + "0.8": 0.8816000442161055, + "0.9": 0.9232743036857335 + }, + "matched_mcc_abs": { + "0.6": 0.8010895625130967, + "0.7": 0.8418584092295176, + "0.8": 0.8814625852652623, + "0.9": 0.9230757036865166 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5866230160752731, + "mcc_abs": 0.5866534820280306, + "act_iou": 0.25410334270062807, + "act_iou_n_pairs": 12288, + "input_r2": 0.9568342577163039, + "matched_mcc_signed": { + "0.6": 0.8015788475991067, + "0.7": 0.8403394457545333, + "0.8": 0.8777995993247695, + "0.9": 0.9204097267206155 + }, + "matched_mcc_abs": { + "0.6": 0.7978128005805744, + "0.7": 0.8392514719887179, + "0.8": 0.8775768445716118, + "0.9": 0.9200439513126423 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 64.37366222222222, + "l0_std": 0.22549914148847303, + "recon_r2_mean": 0.9273735981584638, + "recon_r2_std": 0.00020824504809352376, + "l1_per_token_mean": 86.46451331629923, + "l1_per_token_std": 0.2873837616368954, + "input_r2_mean": 0.9569485191327898, + "input_r2_std": 0.0005311355823059143, + "mcc_signed_mean": 0.5869038452870696, + "mcc_signed_std": 0.00026248986763339636, + "mcc_abs_mean": 0.5869341092094363, + "mcc_abs_std": 0.00026186314439693217, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8027545926770494, + "0.7": 0.8411107135340865, + "0.8": 0.8795645134236135, + "0.9": 0.9231112500199603 + }, + "matched_mcc_signed_band": 0.8616352674136775, + "matched_mcc_abs_per_threshold": { + "0.6": 0.7994063953203452, + "0.7": 0.8402895653860103, + "0.8": 0.8795659242503953, + "0.9": 0.9231533325817103 + }, + "matched_mcc_abs_band": 0.8606038043846153, + "act_iou_mean": 0.2549716906210498, + "act_iou_std": 0.0008002282825589148, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/dict_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..a39e32e0ef27be651d0afbd331808581e8fd1a74 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 64.05448647460938, + "recon_r2": 0.9277306652808159, + "l1_per_token": 86.16258715545342, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 96, + "dense_rate": 0.0078125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 4, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 4, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.185712531208992, + "n_dup_abs": { + "0.5": 446, + "0.6": 310, + "0.7": 262, + "0.8": 208, + "0.9": 170 + }, + "n_dup_clusters_abs": { + "0.5": 223, + "0.6": 155, + "0.7": 131, + "0.8": 104, + "0.9": 85 + }, + "n_dup_removed_abs": { + "0.5": 223, + "0.6": 155, + "0.7": 131, + "0.8": 104, + "0.9": 85 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2518947422504425, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 64.37109682617188, + "recon_r2": 0.9274358205176751, + "l1_per_token": 86.45983637496936, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 91, + "dense_rate": 0.007405598958333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 10, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 5, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 5, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.18523038923740387, + "n_dup_abs": { + "0.5": 450, + "0.6": 324, + "0.7": 252, + "0.8": 206, + "0.9": 162 + }, + "n_dup_clusters_abs": { + "0.5": 225, + "0.6": 162, + "0.7": 126, + "0.8": 103, + "0.9": 81 + }, + "n_dup_removed_abs": { + "0.5": 225, + "0.6": 162, + "0.7": 126, + "0.8": 103, + "0.9": 81 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.25195565819740295, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": 64, + "l0_raw": 64.08063813476562, + "recon_r2": 0.951196823582813, + "l1_per_token": 85.96976629743486, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 94, + "dense_rate": 0.007649739583333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 12, + "0.6": 4, + "0.7": 2, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 6, + "0.6": 2, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 6, + "0.6": 2, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.18519680202007294, + "n_dup_abs": { + "0.5": 428, + "0.6": 300, + "0.7": 236, + "0.8": 202, + "0.9": 160 + }, + "n_dup_clusters_abs": { + "0.5": 214, + "0.6": 150, + "0.7": 118, + "0.8": 101, + "0.9": 80 + }, + "n_dup_removed_abs": { + "0.5": 214, + "0.6": 150, + "0.7": 118, + "0.8": 101, + "0.9": 80 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.25079870223999023, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.5927438139915466, + "dcs_abs": 0.5962637066841125, + "frac_sign_flipped": 0.06331380208333333, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.5926756262779236, + "0.6": 0.5927848219871521, + "0.7": 0.5927438139915466, + "0.8": 0.5927438139915466, + "0.9": 0.5927438139915466 + }, + "unique_frac_signed": { + "0.5": 0.9996337890625, + "0.6": 0.9999593098958334, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.592458581981191, + "0.6": 0.592760701510997, + "0.7": 0.5927438139915466, + "0.8": 0.5927438139915466, + "0.9": 0.5927438139915466 + }, + "matched_frac_signed": { + "0.5": 0.66650390625, + "0.6": 0.5524088541666666, + "0.7": 0.4146321614583333, + "0.8": 0.24788411458333334, + "0.9": 0.07568359375 + }, + "matched_cos_abs": { + "0.5": 0.595612645149231, + "0.6": 0.595893919467926, + "0.7": 0.5959410071372986, + "0.8": 0.5958201289176941, + "0.9": 0.5960332155227661 + }, + "unique_frac_abs": { + "0.5": 0.9817708333333334, + "0.6": 0.9871012369791666, + "0.7": 0.9895426432291666, + "0.8": 0.9915771484375, + "0.9": 0.9932454427083334 + }, + "dedup_dcs_abs": { + "0.5": 0.5847551229720315, + "0.6": 0.5882076250151537, + "0.7": 0.589709039411294, + "0.8": 0.5908016244138707, + "0.9": 0.5920072750207813 + }, + "matched_frac_abs": { + "0.5": 0.6582845052083334, + "0.6": 0.5486653645833334, + "0.7": 0.4109700520833333, + "0.8": 0.2451171875, + "0.9": 0.07413736979166667 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.5934622287750244, + "dcs_abs": 0.5973575711250305, + "frac_sign_flipped": 0.06599934895833333, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.5934563279151917, + "0.6": 0.5935021042823792, + "0.7": 0.5935021638870239, + "0.8": 0.5934622287750244, + "0.9": 0.5934622287750244 + }, + "unique_frac_signed": { + "0.5": 0.9995930989583334, + "0.6": 0.9999186197916666, + "0.7": 0.9999593098958334, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5932148499171793, + "0.6": 0.5934538049574863, + "0.7": 0.5934780142221523, + "0.8": 0.5934622287750244, + "0.9": 0.5934622287750244 + }, + "matched_frac_signed": { + "0.5": 0.66357421875, + "0.6": 0.5576985677083334, + "0.7": 0.4183756510416667, + "0.8": 0.2500813802083333, + "0.9": 0.08186848958333333 + }, + "matched_cos_abs": { + "0.5": 0.5970430970191956, + "0.6": 0.597295880317688, + "0.7": 0.5974531173706055, + "0.8": 0.5970664024353027, + "0.9": 0.5970839858055115 + }, + "unique_frac_abs": { + "0.5": 0.9822184244791666, + "0.6": 0.9875895182291666, + "0.7": 0.9898681640625, + "0.8": 0.9916585286458334, + "0.9": 0.9932861328125 + }, + "dedup_dcs_abs": { + "0.5": 0.5864267301003565, + "0.6": 0.5898831506832114, + "0.7": 0.5913998204050586, + "0.8": 0.5920859901428533, + "0.9": 0.5930752432250301 + }, + "matched_frac_abs": { + "0.5": 0.65478515625, + "0.6": 0.5538736979166666, + "0.7": 0.4150390625, + "0.8": 0.24755859375, + "0.9": 0.08048502604166667 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.591376006603241, + "dcs_abs": 0.5952784419059753, + "frac_sign_flipped": 0.06852213541666667, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.5913077592849731, + "0.6": 0.5914262533187866, + "0.7": 0.5913981795310974, + "0.8": 0.591376006603241, + "0.9": 0.591376006603241 + }, + "unique_frac_signed": { + "0.5": 0.9995524088541666, + "0.6": 0.9998779296875, + "0.7": 0.9999593098958334, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5910430951674546, + "0.6": 0.5913540577312233, + "0.7": 0.5913741154775684, + "0.8": 0.591376006603241, + "0.9": 0.591376006603241 + }, + "matched_frac_signed": { + "0.5": 0.664306640625, + "0.6": 0.5530598958333334, + "0.7": 0.4103190104166667, + "0.8": 0.242431640625, + "0.9": 0.07584635416666667 + }, + "matched_cos_abs": { + "0.5": 0.5947926640510559, + "0.6": 0.5952449440956116, + "0.7": 0.595166027545929, + "0.8": 0.5950164198875427, + "0.9": 0.5949280261993408 + }, + "unique_frac_abs": { + "0.5": 0.9821370442708334, + "0.6": 0.9873046875, + "0.7": 0.9900716145833334, + "0.8": 0.99169921875, + "0.9": 0.9934488932291666 + }, + "dedup_dcs_abs": { + "0.5": 0.5841679090250789, + "0.6": 0.5876881235162728, + "0.7": 0.5892569898375466, + "0.8": 0.5900773187458981, + "0.9": 0.5910305891787477 + }, + "matched_frac_abs": { + "0.5": 0.6582845052083334, + "0.6": 0.550537109375, + "0.7": 0.4073079427083333, + "0.8": 0.23990885416666666, + "0.9": 0.07438151041666667 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 94, + "n_dense_std": 2.516611478423583, + "l0_mean": 64, + "l0_std": 0.17573288226685352, + "recon_r2_mean": 0.9354544364604346, + "recon_r2_std": 0.013634104209360601, + "l1_per_token_mean": 86.19739660928587, + "l1_per_token_std": 0.24688244931698258, + "babel_mu1_signed_mean": 0.18537990748882294, + "babel_mu1_signed_std": 0.0002885497005826576, + "n_dup_signed_per_threshold": { + "0.5": 10.0, + "0.6": 2.0, + "0.7": 0.6666666666666666, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.6666666666666666, + "n_dup_signed_band_std": 0.7637626158259734, + "n_dup_clusters_signed_per_threshold": { + "0.5": 5.0, + "0.6": 1.0, + "0.7": 0.3333333333333333, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.3333333333333333, + "n_dup_clusters_signed_band_std": 0.3818813079129867, + "largest_dup_signed_per_threshold": { + "0.5": 2.0, + "0.6": 1.6666666666666667, + "0.7": 1.3333333333333333, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.25, + "largest_dup_signed_band_std": 0.25, + "dcs_signed_mean": 0.5925273497899374, + "dcs_signed_std": 0.0010598222966346634, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.592238842355275, + "0.6": 0.5925228547332355, + "0.7": 0.5925319812304224, + "0.8": 0.5925273497899374, + "0.9": 0.5925273497899374 + }, + "dedup_dcs_signed_band": 0.5925273838858832, + "dedup_dcs_signed_band_std": 0.001064306400414243, + "matched_cos_signed_per_threshold": { + "0.5": 0.5924799044926962, + "0.6": 0.5925710598627726, + "0.7": 0.5925480524698893, + "0.8": 0.5925273497899374, + "0.9": 0.5925273497899374 + }, + "matched_cos_signed_band": 0.5925434529781342, + "matched_cos_signed_band_std": 0.0010598478070008054, + "unique_frac_signed_per_threshold": { + "0.5": 0.9995930989583334, + "0.6": 0.9999186197916666, + "0.7": 0.999972873263889, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9999728732638888, + "unique_frac_signed_band_std": 1.5538790198286375e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.664794921875, + "0.6": 0.5543891059027778, + "0.7": 0.4144422743055556, + "0.8": 0.24679904513888887, + "0.9": 0.07779947916666667 + }, + "matched_frac_signed_band": 0.32335747612847227, + "matched_frac_signed_band_std": 0.0033520184120944185, + "babel_mu1_abs_mean": 0.25154970089594525, + "babel_mu1_abs_std": 0.0006510967087024637, + "n_dup_abs_per_threshold": { + "0.5": 441.3333333333333, + "0.6": 311.3333333333333, + "0.7": 250.0, + "0.8": 205.33333333333334, + "0.9": 164.0 + }, + "n_dup_abs_band": 232.66666666666666, + "n_dup_abs_band_std": 7.112196097783956, + "n_dup_clusters_abs_per_threshold": { + "0.5": 220.66666666666666, + "0.6": 155.66666666666666, + "0.7": 125.0, + "0.8": 102.66666666666667, + "0.9": 82.0 + }, + "n_dup_clusters_abs_band": 116.33333333333333, + "n_dup_clusters_abs_band_std": 3.556098048891978, + "largest_dup_abs_per_threshold": { + "0.5": 2.0, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.0, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.5962999065717062, + "dcs_abs_std": 0.0010400372114874484, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5851165873658223, + "0.6": 0.5885929664048793, + "0.7": 0.5901219498846331, + "0.8": 0.5909883111008741, + "0.9": 0.5920377024748531 + }, + "dedup_dcs_abs_band": 0.59043523246631, + "dedup_dcs_abs_band_std": 0.0010716871882734394, + "matched_cos_abs_per_threshold": { + "0.5": 0.5958161354064941, + "0.6": 0.5961449146270752, + "0.7": 0.5961867173512777, + "0.8": 0.5959676504135132, + "0.9": 0.5960150758425394 + }, + "matched_cos_abs_band": 0.5960785895586014, + "matched_cos_abs_band_std": 0.001076563892364361, + "unique_frac_abs_per_threshold": { + "0.5": 0.9820421006944445, + "0.6": 0.987331814236111, + "0.7": 0.9898274739583334, + "0.8": 0.9916449652777778, + "0.9": 0.9933268229166666 + }, + "unique_frac_abs_band": 0.9905327690972222, + "unique_frac_abs_band_std": 0.00014469800003631594, + "matched_frac_abs_per_threshold": { + "0.5": 0.6571180555555557, + "0.6": 0.551025390625, + "0.7": 0.41110568576388884, + "0.8": 0.2441948784722222, + "0.9": 0.07633463541666667 + }, + "matched_frac_abs_band": 0.3206651475694445, + "matched_frac_abs_band_std": 0.003208223986448048, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 12288 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed0/config.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3e187a07c65dcae7ddde84a817fa7d9110867df5 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed0/lock.lock b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed0/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..b91dd60fb70952d0c691fb9eb1e7655ac0366fbc --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.05448647460938, + "l0_per_token_0.001": 64.05448647460938, + "l0_per_token_0.01": 64.05448647460938, + "l0_per_token_0.05": 64.05448647460938, + "l0_per_token_0.1": 64.05448647460938, + "l0_per_token_0.5": 64.05448647460938, + "l0_per_token_1.0": 25.16168251953125, + "l1_per_token": 86.16258715545342, + "r2_score": 0.9277306652808159, + "l2_loss": 0.04811482637285759 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed1/config.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..abd948c4d6731bf89a82b1bc8e5174ba3cf4f99a --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed1/lock.lock b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed1/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..d716c500e88fc2cf670b9dc47dc4f0009e8bea60 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.37109682617188, + "l0_per_token_0.001": 64.37109682617188, + "l0_per_token_0.01": 64.37109682617188, + "l0_per_token_0.05": 64.37109682617188, + "l0_per_token_0.1": 64.37109682617188, + "l0_per_token_0.5": 64.37109682617188, + "l0_per_token_1.0": 25.24388955078125, + "l1_per_token": 86.45983637496936, + "r2_score": 0.9274358205176751, + "l2_loss": 0.04831112546209909 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed2/config.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c7856662ca76e8aa6b7e6e38f8b93b4b57f646de --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "768,768,1536,3072,6144", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed2/lock.lock b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed2/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..ae5e7765ef5c230cccc91dc9d5ef0313125a36da --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.08063813476562, + "l0_per_token_0.001": 64.08063813476562, + "l0_per_token_0.01": 64.08063813476562, + "l0_per_token_0.05": 64.08063813476562, + "l0_per_token_0.1": 64.08063813476562, + "l0_per_token_0.5": 64.08063813476562, + "l0_per_token_1.0": 25.176251513671875, + "l1_per_token": 85.96976629743486, + "r2_score": 0.951196823582813, + "l2_loss": 0.04804796836933946 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_128/code_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..3f12b9390affc85606020672381906458f96f52f --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 128.43953, + "recon_r2": 0.9348825085681763, + "l1_per_token": 137.4873661661016 + }, + "seed1": { + "l0": 128.71809, + "recon_r2": 0.9345409202147403, + "l1_per_token": 137.6621760569862 + }, + "seed2": { + "l0": 128.59635833333334, + "recon_r2": 0.9348203386277572, + "l1_per_token": 137.8867248210671 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.593398753748054, + "mcc_abs": 0.5934505769648769, + "act_iou": 0.27811743264748096, + "act_iou_n_pairs": 3072, + "input_r2": 0.9408067457919368, + "matched_mcc_signed": { + "0.6": 0.806131501395926, + "0.7": 0.8397166934044488, + "0.8": 0.8786355729886123, + "0.9": 0.9439359236940776 + }, + "matched_mcc_abs": { + "0.6": 0.8080433016720541, + "0.7": 0.8427265993356211, + "0.8": 0.882488596841991, + "0.9": 0.9421301834871865 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5902397118478545, + "mcc_abs": 0.5903069701926711, + "act_iou": 0.2774675066680696, + "act_iou_n_pairs": 3072, + "input_r2": 0.9411227792966286, + "matched_mcc_signed": { + "0.6": 0.8067465313620721, + "0.7": 0.8395811430942907, + "0.8": 0.8773528701231329, + "0.9": 0.9362813184413609 + }, + "matched_mcc_abs": { + "0.6": 0.8067779995555882, + "0.7": 0.8418324700218447, + "0.8": 0.8834329890093283, + "0.9": 0.9355363900450444 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5876223924770949, + "mcc_abs": 0.5876897971784943, + "act_iou": 0.2758529072981682, + "act_iou_n_pairs": 3072, + "input_r2": 0.9405821766306119, + "matched_mcc_signed": { + "0.6": 0.8039899001218469, + "0.7": 0.8361660469113823, + "0.8": 0.8792796942143251, + "0.9": 0.9251014780487842 + }, + "matched_mcc_abs": { + "0.6": 0.8063315920399048, + "0.7": 0.8384309621592129, + "0.8": 0.8816426473482817, + "0.9": 0.9307686657572938 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 128.58465944444444, + "l0_std": 0.13964800894007037, + "recon_r2_mean": 0.9347479224702245, + "recon_r2_std": 0.0001819442928607934, + "l1_per_token_mean": 137.67875568138496, + "l1_per_token_std": 0.20019489699507478, + "input_r2_mean": 0.9408372339063925, + "input_r2_std": 0.00027158783932342744, + "mcc_signed_mean": 0.5904202860243345, + "mcc_signed_std": 0.002892411218701007, + "mcc_abs_mean": 0.5904824481120142, + "mcc_abs_std": 0.002884396004700326, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8056226442932818, + "0.7": 0.8384879611367072, + "0.8": 0.8784227124420235, + "0.9": 0.9351062400614075 + }, + "matched_mcc_signed_band": 0.864409889483355, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8070509644225158, + "0.7": 0.8409966771722263, + "0.8": 0.8825214110665337, + "0.9": 0.9361450797631748 + }, + "matched_mcc_abs_band": 0.8666785331061126, + "act_iou_mean": 0.27714594887123956, + "act_iou_std": 0.0011660052859857218, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_128/dict_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..8fe5ebaa725dc21326ac5738cd7efbd07ae2ad22 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 128, + "l0_raw": 128.13727158203125, + "recon_r2": 0.9350740243697656, + "l1_per_token": 137.13712130599035, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 344, + "dense_rate": 0.11197916666666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.14246203005313873, + "n_dup_abs": { + "0.5": 608, + "0.6": 568, + "0.7": 546, + "0.8": 534, + "0.9": 480 + }, + "n_dup_clusters_abs": { + "0.5": 304, + "0.6": 284, + "0.7": 273, + "0.8": 267, + "0.9": 240 + }, + "n_dup_removed_abs": { + "0.5": 304, + "0.6": 284, + "0.7": 273, + "0.8": 267, + "0.9": 240 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.35510098934173584, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 128, + "l0_raw": 128.326883984375, + "recon_r2": 0.9347642539455845, + "l1_per_token": 137.24203858373733, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 361, + "dense_rate": 0.11751302083333333, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.14201846718788147, + "n_dup_abs": { + "0.5": 624, + "0.6": 592, + "0.7": 574, + "0.8": 558, + "0.9": 512 + }, + "n_dup_clusters_abs": { + "0.5": 312, + "0.6": 296, + "0.7": 287, + "0.8": 279, + "0.9": 256 + }, + "n_dup_removed_abs": { + "0.5": 312, + "0.6": 296, + "0.7": 287, + "0.8": 279, + "0.9": 256 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.36328229308128357, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 128, + "l0_raw": 128.30451381835937, + "recon_r2": 0.9560778283366386, + "l1_per_token": 137.54306221680503, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 344, + "dense_rate": 0.11197916666666667, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.14153388142585754, + "n_dup_abs": { + "0.5": 604, + "0.6": 580, + "0.7": 572, + "0.8": 554, + "0.9": 490 + }, + "n_dup_clusters_abs": { + "0.5": 302, + "0.6": 290, + "0.7": 286, + "0.8": 277, + "0.9": 245 + }, + "n_dup_removed_abs": { + "0.5": 302, + "0.6": 290, + "0.7": 286, + "0.8": 277, + "0.9": 245 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3586009442806244, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.5813858509063721, + "dcs_abs": 0.5847962498664856, + "frac_sign_flipped": 0.08138020833333333, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.5813858509063721, + "0.6": 0.5813858509063721, + "0.7": 0.5813858509063721, + "0.8": 0.5813858509063721, + "0.9": 0.5813858509063721 + }, + "unique_frac_signed": { + "0.5": 1.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5813858509063721, + "0.6": 0.5813858509063721, + "0.7": 0.5813858509063721, + "0.8": 0.5813858509063721, + "0.9": 0.5813858509063721 + }, + "matched_frac_signed": { + "0.5": 0.6634114583333334, + "0.6": 0.5634765625, + "0.7": 0.3994140625, + "0.8": 0.1982421875, + "0.9": 0.055013020833333336 + }, + "matched_cos_abs": { + "0.5": 0.6020811200141907, + "0.6": 0.6027329564094543, + "0.7": 0.6028894782066345, + "0.8": 0.6027060151100159, + "0.9": 0.6013199687004089 + }, + "unique_frac_abs": { + "0.5": 0.8997395833333334, + "0.6": 0.9055989583333334, + "0.7": 0.9088541666666666, + "0.8": 0.9111328125, + "0.9": 0.9192708333333334 + }, + "dedup_dcs_abs": { + "0.5": 0.5417162160544347, + "0.6": 0.5458343374775723, + "0.7": 0.5479386143075923, + "0.8": 0.5491452266578563, + "0.9": 0.5527759087271988 + }, + "matched_frac_abs": { + "0.5": 0.6318359375, + "0.6": 0.5403645833333334, + "0.7": 0.3828125, + "0.8": 0.18880208333333334, + "0.9": 0.052408854166666664 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.5773628354072571, + "dcs_abs": 0.5813533663749695, + "frac_sign_flipped": 0.08170572916666667, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.5773628354072571, + "0.6": 0.5773628354072571, + "0.7": 0.5773628354072571, + "0.8": 0.5773628354072571, + "0.9": 0.5773628354072571 + }, + "unique_frac_signed": { + "0.5": 1.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5773628354072571, + "0.6": 0.5773628354072571, + "0.7": 0.5773628354072571, + "0.8": 0.5773628354072571, + "0.9": 0.5773628354072571 + }, + "matched_frac_signed": { + "0.5": 0.6650390625, + "0.6": 0.5576171875, + "0.7": 0.38671875, + "0.8": 0.19694010416666666, + "0.9": 0.0537109375 + }, + "matched_cos_abs": { + "0.5": 0.5998122692108154, + "0.6": 0.5999410152435303, + "0.7": 0.6004945635795593, + "0.8": 0.599726140499115, + "0.9": 0.5963556170463562 + }, + "unique_frac_abs": { + "0.5": 0.9013671875, + "0.6": 0.9065755208333334, + "0.7": 0.9090169270833334, + "0.8": 0.9114583333333334, + "0.9": 0.9210611979166666 + }, + "dedup_dcs_abs": { + "0.5": 0.5406510981265455, + "0.6": 0.5438918383636823, + "0.7": 0.5458597229153384, + "0.8": 0.5466253884757559, + "0.9": 0.5492800190210497 + }, + "matched_frac_abs": { + "0.5": 0.6370442708333334, + "0.6": 0.5406901041666666, + "0.7": 0.3717447916666667, + "0.8": 0.18522135416666666, + "0.9": 0.051106770833333336 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.5759053826332092, + "dcs_abs": 0.579194962978363, + "frac_sign_flipped": 0.0771484375, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.5759053826332092, + "0.6": 0.5759053826332092, + "0.7": 0.5759053826332092, + "0.8": 0.5759053826332092, + "0.9": 0.5759053826332092 + }, + "unique_frac_signed": { + "0.5": 1.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5759053826332092, + "0.6": 0.5759053826332092, + "0.7": 0.5759053826332092, + "0.8": 0.5759053826332092, + "0.9": 0.5759053826332092 + }, + "matched_frac_signed": { + "0.5": 0.6552734375, + "0.6": 0.548828125, + "0.7": 0.390625, + "0.8": 0.19010416666666666, + "0.9": 0.0615234375 + }, + "matched_cos_abs": { + "0.5": 0.5969855785369873, + "0.6": 0.5968095660209656, + "0.7": 0.5960095524787903, + "0.8": 0.5959952473640442, + "0.9": 0.5944017767906189 + }, + "unique_frac_abs": { + "0.5": 0.9000651041666666, + "0.6": 0.9046223958333334, + "0.7": 0.90673828125, + "0.8": 0.9095052083333334, + "0.9": 0.91845703125 + }, + "dedup_dcs_abs": { + "0.5": 0.5373258869318912, + "0.6": 0.5398872994701378, + "0.7": 0.5404246772232, + "0.8": 0.5420607816195115, + "0.9": 0.545932491280837 + }, + "matched_frac_abs": { + "0.5": 0.6214192708333334, + "0.6": 0.5276692708333334, + "0.7": 0.3743489583333333, + "0.8": 0.17936197916666666, + "0.9": 0.056315104166666664 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 350, + "n_dense_std": 9.814954576223638, + "l0_mean": 128, + "l0_std": 0.10362050397336772, + "recon_r2_mean": 0.9419720355506628, + "recon_r2_std": 0.012216956741332392, + "l1_per_token_mean": 137.30740736884425, + "l1_per_token_std": 0.2107173801992313, + "babel_mu1_signed_mean": 0.14200479288895926, + "babel_mu1_signed_std": 0.0004642253853671184, + "n_dup_signed_per_threshold": { + "0.5": 0.0, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.0, + "n_dup_signed_band_std": 0.0, + "n_dup_clusters_signed_per_threshold": { + "0.5": 0.0, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.0, + "n_dup_clusters_signed_band_std": 0.0, + "largest_dup_signed_per_threshold": { + "0.5": 1.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.0, + "largest_dup_signed_band_std": 0.0, + "dcs_signed_mean": 0.5782180229822794, + "dcs_signed_std": 0.0028385546435889903, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5782180229822794, + "0.6": 0.5782180229822794, + "0.7": 0.5782180229822794, + "0.8": 0.5782180229822794, + "0.9": 0.5782180229822794 + }, + "dedup_dcs_signed_band": 0.5782180229822794, + "dedup_dcs_signed_band_std": 0.0028385546435889903, + "matched_cos_signed_per_threshold": { + "0.5": 0.5782180229822794, + "0.6": 0.5782180229822794, + "0.7": 0.5782180229822794, + "0.8": 0.5782180229822794, + "0.9": 0.5782180229822794 + }, + "matched_cos_signed_band": 0.5782180229822794, + "matched_cos_signed_band_std": 0.0028385546435889903, + "unique_frac_signed_per_threshold": { + "0.5": 1.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 1.0, + "unique_frac_signed_band_std": 0.0, + "matched_frac_signed_per_threshold": { + "0.5": 0.6612413194444445, + "0.6": 0.556640625, + "0.7": 0.3922526041666667, + "0.8": 0.19509548611111108, + "0.9": 0.05674913194444445 + }, + "matched_frac_signed_band": 0.3001844618055556, + "matched_frac_signed_band_std": 0.0033714724552649784, + "babel_mu1_abs_mean": 0.3589947422345479, + "babel_mu1_abs_std": 0.004104843522117523, + "n_dup_abs_per_threshold": { + "0.5": 612.0, + "0.6": 580.0, + "0.7": 564.0, + "0.8": 548.6666666666666, + "0.9": 494.0 + }, + "n_dup_abs_band": 546.6666666666666, + "n_dup_abs_band_std": 13.650396819628847, + "n_dup_clusters_abs_per_threshold": { + "0.5": 306.0, + "0.6": 290.0, + "0.7": 282.0, + "0.8": 274.3333333333333, + "0.9": 247.0 + }, + "n_dup_clusters_abs_band": 273.3333333333333, + "n_dup_clusters_abs_band_std": 6.825198409814424, + "largest_dup_abs_per_threshold": { + "0.5": 2.0, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.0, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.5817815264066061, + "dcs_abs_std": 0.002825083089093432, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5398977337042905, + "0.6": 0.5432044917704641, + "0.7": 0.5447410048153768, + "0.8": 0.545943798917708, + "0.9": 0.5493294730096951 + }, + "dedup_dcs_abs_band": 0.545804692128311, + "dedup_dcs_abs_band_std": 0.003464063016700656, + "matched_cos_abs_per_threshold": { + "0.5": 0.5996263225873312, + "0.6": 0.5998278458913168, + "0.7": 0.5997978647549947, + "0.8": 0.5994758009910583, + "0.9": 0.5973591208457947 + }, + "matched_cos_abs_band": 0.5991151581207912, + "matched_cos_abs_band_std": 0.00330405727970344, + "unique_frac_abs_per_threshold": { + "0.5": 0.900390625, + "0.6": 0.9055989583333334, + "0.7": 0.908203125, + "0.8": 0.9106987847222223, + "0.9": 0.9195963541666666 + }, + "unique_frac_abs_band": 0.9110243055555556, + "unique_frac_abs_band_std": 0.001110872137014013, + "matched_frac_abs_per_threshold": { + "0.5": 0.630099826388889, + "0.6": 0.5362413194444445, + "0.7": 0.3763020833333333, + "0.8": 0.18446180555555555, + "0.9": 0.05327690972222222 + }, + "matched_frac_abs_band": 0.28757052951388895, + "matched_frac_abs_band_std": 0.0033527592351302845, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 3072 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed0/config.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..302eee48c766f1dcdb2d85c3c4cecf838b66a3d7 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..51d24de99208a7208b3209d79690e517ac471e66 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.13727158203125, + "l0_per_token_0.001": 128.13727158203125, + "l0_per_token_0.01": 128.13727158203125, + "l0_per_token_0.05": 128.13727158203125, + "l0_per_token_0.1": 128.13727158203125, + "l0_per_token_0.5": 108.7157931640625, + "l0_per_token_1.0": 36.453852734375, + "l1_per_token": 137.13712130599035, + "r2_score": 0.9350740243697656, + "l2_loss": 0.043225830937500855 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed1/config.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..aadd6db1df53ffb0cbe4db859cda6475f2dea8fe --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..58b8088e8d14a5299ace4511861f34f49ecbe1ef --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.326883984375, + "l0_per_token_0.001": 128.326883984375, + "l0_per_token_0.01": 128.326883984375, + "l0_per_token_0.05": 128.326883984375, + "l0_per_token_0.1": 128.326883984375, + "l0_per_token_0.5": 109.27421010742188, + "l0_per_token_1.0": 36.13624755859375, + "l1_per_token": 137.24203858373733, + "r2_score": 0.9347642539455845, + "l2_loss": 0.04343206709883871 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed2/config.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f453baeacffb08d7c2804e003c32605c9dd3ba1b --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e4fa2d841e5b889dc2f77246056b7cfceaebd84a --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.30451381835937, + "l0_per_token_0.001": 128.30451381835937, + "l0_per_token_0.01": 128.30451381835937, + "l0_per_token_0.05": 128.30451381835937, + "l0_per_token_0.1": 128.30451381835937, + "l0_per_token_0.5": 109.1313083984375, + "l0_per_token_1.0": 36.518371435546875, + "l1_per_token": 137.54306221680503, + "r2_score": 0.9560778283366386, + "l2_loss": 0.043242495053061276 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_32/code_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..cfa65f86ebf61bcd42afa8111fddff2c093cd635 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 32.081965, + "recon_r2": 0.8863913596159557, + "l1_per_token": 66.65286854979128 + }, + "seed1": { + "l0": 32.14357833333333, + "recon_r2": 0.8857690796931421, + "l1_per_token": 66.7898777971055 + }, + "seed2": { + "l0": 32.167705, + "recon_r2": 0.8863640523698502, + "l1_per_token": 66.89140612834781 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6759922293895569, + "mcc_abs": 0.6760450086167573, + "act_iou": 0.36495407994036294, + "act_iou_n_pairs": 3072, + "input_r2": 0.9509109477137132, + "matched_mcc_signed": { + "0.6": 0.8136653506434214, + "0.7": 0.8394878383204329, + "0.8": 0.8724650184200058, + "0.9": 0.9218931660677989 + }, + "matched_mcc_abs": { + "0.6": 0.8112558675282707, + "0.7": 0.8391482758026623, + "0.8": 0.8726805976264503, + "0.9": 0.9216481290716747 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6726586367796492, + "mcc_abs": 0.6727105774741645, + "act_iou": 0.3643890552996931, + "act_iou_n_pairs": 3072, + "input_r2": 0.9512817975370519, + "matched_mcc_signed": { + "0.6": 0.8130983835848961, + "0.7": 0.8432799718985651, + "0.8": 0.8775095709627563, + "0.9": 0.9234295280063084 + }, + "matched_mcc_abs": { + "0.6": 0.8085184816330644, + "0.7": 0.8417504514211553, + "0.8": 0.8770939488381444, + "0.9": 0.9229740931573515 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6758582532228621, + "mcc_abs": 0.6759028956562689, + "act_iou": 0.3625010562338474, + "act_iou_n_pairs": 3072, + "input_r2": 0.950139158728694, + "matched_mcc_signed": { + "0.6": 0.8112541331544149, + "0.7": 0.8394124016830665, + "0.8": 0.8739459645910949, + "0.9": 0.9199182164814951 + }, + "matched_mcc_abs": { + "0.6": 0.8074055673906345, + "0.7": 0.8385977191585207, + "0.8": 0.8739448933807688, + "0.9": 0.9194774705176513 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 32.13108277777777, + "l0_std": 0.04421471566663675, + "recon_r2_mean": 0.8861748305596494, + "recon_r2_std": 0.000351655720380014, + "l1_per_token_mean": 66.77805082508154, + "l1_per_token_std": 0.11970777771868144, + "input_r2_mean": 0.9507773013264863, + "input_r2_std": 0.0005829252775488915, + "mcc_signed_mean": 0.6748363731306894, + "mcc_signed_std": 0.0018871643050598376, + "mcc_abs_mean": 0.6748861605823969, + "mcc_abs_std": 0.0018854496596314417, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8126726224609108, + "0.7": 0.8407267373006881, + "0.8": 0.8746401846579523, + "0.9": 0.9217469701852008 + }, + "matched_mcc_signed_band": 0.862446628651188, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8090599721839898, + "0.7": 0.8398321487941128, + "0.8": 0.8745731466151212, + "0.9": 0.9213665642488924 + }, + "matched_mcc_abs_band": 0.8612079579605291, + "act_iou_mean": 0.36394806382463446, + "act_iou_std": 0.0012845958263156705, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_32/dict_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..15966418d4a10492d2be89b96f9e0e2fcbcbf0cc --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 32, + "l0_raw": 31.98938828125, + "recon_r2": 0.8867659732476206, + "l1_per_token": 66.4808615756082, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 50, + "dense_rate": 0.016276041666666668, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.19231319427490234, + "n_dup_abs": { + "0.5": 128, + "0.6": 78, + "0.7": 52, + "0.8": 34, + "0.9": 22 + }, + "n_dup_clusters_abs": { + "0.5": 64, + "0.6": 39, + "0.7": 26, + "0.8": 17, + "0.9": 11 + }, + "n_dup_removed_abs": { + "0.5": 64, + "0.6": 39, + "0.7": 26, + "0.8": 17, + "0.9": 11 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.26423558592796326, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 32, + "l0_raw": 32.062194140625, + "recon_r2": 0.8862758749591823, + "l1_per_token": 66.6345748946161, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 51, + "dense_rate": 0.0166015625, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.19149065017700195, + "n_dup_abs": { + "0.5": 104, + "0.6": 66, + "0.7": 54, + "0.8": 34, + "0.9": 20 + }, + "n_dup_clusters_abs": { + "0.5": 52, + "0.6": 33, + "0.7": 27, + "0.8": 17, + "0.9": 10 + }, + "n_dup_removed_abs": { + "0.5": 52, + "0.6": 33, + "0.7": 27, + "0.8": 17, + "0.9": 10 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2623380720615387, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 32, + "l0_raw": 32.0752048828125, + "recon_r2": 0.9234362244403416, + "l1_per_token": 66.70951532040115, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 46, + "dense_rate": 0.014973958333333334, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.19154423475265503, + "n_dup_abs": { + "0.5": 128, + "0.6": 84, + "0.7": 52, + "0.8": 38, + "0.9": 22 + }, + "n_dup_clusters_abs": { + "0.5": 64, + "0.6": 42, + "0.7": 26, + "0.8": 19, + "0.9": 11 + }, + "n_dup_removed_abs": { + "0.5": 64, + "0.6": 42, + "0.7": 26, + "0.8": 19, + "0.9": 11 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2656302750110626, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.7096638083457947, + "dcs_abs": 0.7135183811187744, + "frac_sign_flipped": 0.0556640625, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7097896933555603, + "0.6": 0.7096638083457947, + "0.7": 0.7096638083457947, + "0.8": 0.7096638083457947, + "0.9": 0.7096638083457947 + }, + "unique_frac_signed": { + "0.5": 0.9996744791666666, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7095586420230877, + "0.6": 0.7096638083457947, + "0.7": 0.7096638083457947, + "0.8": 0.7096638083457947, + "0.9": 0.7096638083457947 + }, + "matched_frac_signed": { + "0.5": 0.8167317708333334, + "0.6": 0.7477213541666666, + "0.7": 0.650390625, + "0.8": 0.4915364583333333, + "0.9": 0.21875 + }, + "matched_cos_abs": { + "0.5": 0.7148698568344116, + "0.6": 0.7140860557556152, + "0.7": 0.7136751413345337, + "0.8": 0.7131041288375854, + "0.9": 0.7130809426307678 + }, + "unique_frac_abs": { + "0.5": 0.9811197916666666, + "0.6": 0.98828125, + "0.7": 0.9913736979166666, + "0.8": 0.9944661458333334, + "0.9": 0.99658203125 + }, + "dedup_dcs_abs": { + "0.5": 0.7013729650061578, + "0.6": 0.7057178597897291, + "0.7": 0.7075187639760163, + "0.8": 0.7091579145829504, + "0.9": 0.7106436542526353 + }, + "matched_frac_abs": { + "0.5": 0.8053385416666666, + "0.6": 0.7425130208333334, + "0.7": 0.6461588541666666, + "0.8": 0.4879557291666667, + "0.9": 0.216796875 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.7069733738899231, + "dcs_abs": 0.7108767032623291, + "frac_sign_flipped": 0.052734375, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7070477604866028, + "0.6": 0.7070477604866028, + "0.7": 0.7069733738899231, + "0.8": 0.7069733738899231, + "0.9": 0.7069733738899231 + }, + "unique_frac_signed": { + "0.5": 0.9998372395833334, + "0.6": 0.9998372395833334, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7069326810985028, + "0.6": 0.7069326810985028, + "0.7": 0.7069733738899231, + "0.8": 0.7069733738899231, + "0.9": 0.7069733738899231 + }, + "matched_frac_signed": { + "0.5": 0.8108723958333334, + "0.6": 0.7366536458333334, + "0.7": 0.6344401041666666, + "0.8": 0.4895833333333333, + "0.9": 0.2275390625 + }, + "matched_cos_abs": { + "0.5": 0.7117180228233337, + "0.6": 0.7109864950180054, + "0.7": 0.7104615569114685, + "0.8": 0.710679829120636, + "0.9": 0.7102158665657043 + }, + "unique_frac_abs": { + "0.5": 0.9791666666666666, + "0.6": 0.98681640625, + "0.7": 0.9915364583333334, + "0.8": 0.994140625, + "0.9": 0.9964192708333334 + }, + "dedup_dcs_abs": { + "0.5": 0.6968905640145142, + "0.6": 0.7016131379059516, + "0.7": 0.7044485359219834, + "0.8": 0.7065156894968823, + "0.9": 0.7076727758976631 + }, + "matched_frac_abs": { + "0.5": 0.796875, + "0.6": 0.7314453125, + "0.7": 0.6315104166666666, + "0.8": 0.4869791666666667, + "0.9": 0.2255859375 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.7088591456413269, + "dcs_abs": 0.712942898273468, + "frac_sign_flipped": 0.0576171875, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7088747024536133, + "0.6": 0.7089303731918335, + "0.7": 0.7088591456413269, + "0.8": 0.7088591456413269, + "0.9": 0.7088591456413269 + }, + "unique_frac_signed": { + "0.5": 0.99951171875, + "0.6": 0.9998372395833334, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.7085285722278059, + "0.6": 0.7088149873889051, + "0.7": 0.7088591456413269, + "0.8": 0.7088591456413269, + "0.9": 0.7088591456413269 + }, + "matched_frac_signed": { + "0.5": 0.8108723958333334, + "0.6": 0.7486979166666666, + "0.7": 0.6500651041666666, + "0.8": 0.4918619791666667, + "0.9": 0.21842447916666666 + }, + "matched_cos_abs": { + "0.5": 0.7145596742630005, + "0.6": 0.7144157290458679, + "0.7": 0.7132775783538818, + "0.8": 0.7128262519836426, + "0.9": 0.7125599384307861 + }, + "unique_frac_abs": { + "0.5": 0.9811197916666666, + "0.6": 0.98779296875, + "0.7": 0.9913736979166666, + "0.8": 0.994140625, + "0.9": 0.99658203125 + }, + "dedup_dcs_abs": { + "0.5": 0.7010686387463162, + "0.6": 0.7056948339159135, + "0.7": 0.7071246304937328, + "0.8": 0.7086495356634259, + "0.9": 0.7101244308287278 + }, + "matched_frac_abs": { + "0.5": 0.8011067708333334, + "0.6": 0.7464192708333334, + "0.7": 0.6481119791666666, + "0.8": 0.4895833333333333, + "0.9": 0.216796875 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 49, + "n_dense_std": 2.6457513110645907, + "l0_mean": 32, + "l0_std": 0.04625015617151788, + "recon_r2_mean": 0.8988260242157148, + "recon_r2_std": 0.021314467279414856, + "l1_per_token_mean": 66.60831726354182, + "l1_per_token_std": 0.11656642374279388, + "babel_mu1_signed_mean": 0.19178269306818643, + "babel_mu1_signed_std": 0.00046020807732182514, + "n_dup_signed_per_threshold": { + "0.5": 2.0, + "0.6": 0.6666666666666666, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.16666666666666666, + "n_dup_signed_band_std": 0.2886751345948129, + "n_dup_clusters_signed_per_threshold": { + "0.5": 1.0, + "0.6": 0.3333333333333333, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.08333333333333333, + "n_dup_clusters_signed_band_std": 0.14433756729740646, + "largest_dup_signed_per_threshold": { + "0.5": 1.6666666666666667, + "0.6": 1.3333333333333333, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.0833333333333333, + "largest_dup_signed_band_std": 0.14433756729740646, + "dcs_signed_mean": 0.7084987759590149, + "dcs_signed_std": 0.0013809450102312182, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.7083399651164655, + "0.6": 0.7084704922777343, + "0.7": 0.7084987759590149, + "0.8": 0.7084987759590149, + "0.9": 0.7084987759590149 + }, + "dedup_dcs_signed_band": 0.7084917050386947, + "dedup_dcs_signed_band_std": 0.0013851305716694946, + "matched_cos_signed_per_threshold": { + "0.5": 0.7085707187652588, + "0.6": 0.708547314008077, + "0.7": 0.7084987759590149, + "0.8": 0.7084987759590149, + "0.9": 0.7084987759590149 + }, + "matched_cos_signed_band": 0.7085109104712805, + "matched_cos_signed_band_std": 0.001373014714895391, + "unique_frac_signed_per_threshold": { + "0.5": 0.9996744791666666, + "0.6": 0.9998914930555557, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.999972873263889, + "unique_frac_signed_band_std": 2.3492442593957545e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.8128255208333334, + "0.6": 0.7443576388888888, + "0.7": 0.6449652777777777, + "0.8": 0.4909939236111111, + "0.9": 0.22157118055555555 + }, + "matched_frac_signed_band": 0.5254720052083333, + "matched_frac_signed_band_std": 0.002961166243271557, + "babel_mu1_abs_mean": 0.26406797766685486, + "babel_mu1_abs_std": 0.0016524888689813753, + "n_dup_abs_per_threshold": { + "0.5": 120.0, + "0.6": 76.0, + "0.7": 52.666666666666664, + "0.8": 35.333333333333336, + "0.9": 21.333333333333332 + }, + "n_dup_abs_band": 46.333333333333336, + "n_dup_abs_band_std": 2.753785273643051, + "n_dup_clusters_abs_per_threshold": { + "0.5": 60.0, + "0.6": 38.0, + "0.7": 26.333333333333332, + "0.8": 17.666666666666668, + "0.9": 10.666666666666666 + }, + "n_dup_clusters_abs_band": 23.166666666666668, + "n_dup_clusters_abs_band_std": 1.3768926368215255, + "largest_dup_abs_per_threshold": { + "0.5": 2.0, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.0, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.7124459942181905, + "dcs_abs_std": 0.0013891726690450682, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6997773892556628, + "0.6": 0.7043419438705314, + "0.7": 0.7063639767972442, + "0.8": 0.7081077132477529, + "0.9": 0.7094802869930087 + }, + "dedup_dcs_abs_band": 0.7070734802271342, + "dedup_dcs_abs_band_std": 0.001750868569307356, + "matched_cos_abs_per_threshold": { + "0.5": 0.7137158513069153, + "0.6": 0.7131627599398295, + "0.7": 0.7124714255332947, + "0.8": 0.7122034033139547, + "0.9": 0.7119522492090861 + }, + "matched_cos_abs_band": 0.7124474594990412, + "matched_cos_abs_band_std": 0.0016157625781861456, + "unique_frac_abs_per_threshold": { + "0.5": 0.98046875, + "0.6": 0.9876302083333334, + "0.7": 0.9914279513888888, + "0.8": 0.9942491319444445, + "0.9": 0.9965277777777778 + }, + "unique_frac_abs_band": 0.9924587673611112, + "unique_frac_abs_band_std": 0.00022410361927429763, + "matched_frac_abs_per_threshold": { + "0.5": 0.8011067708333334, + "0.6": 0.7401258680555557, + "0.7": 0.6419270833333334, + "0.8": 0.4881727430555556, + "0.9": 0.2197265625 + }, + "matched_frac_abs_band": 0.5224880642361112, + "matched_frac_abs_band_std": 0.003261644570257356, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 3072 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed0/config.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3373894456a1d6c6b080169c1b85162d45a7592b --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..a4da56ed3f766804355ab7d20e2b497a42c14f26 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.98938828125, + "l0_per_token_0.001": 31.98938828125, + "l0_per_token_0.01": 31.98938828125, + "l0_per_token_0.05": 31.98938828125, + "l0_per_token_0.1": 31.98938828125, + "l0_per_token_0.5": 31.98938828125, + "l0_per_token_1.0": 24.42100703125, + "l1_per_token": 66.4808615756082, + "r2_score": 0.8867659732476206, + "l2_loss": 0.07538792985794569 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed1/config.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bcc064d33eaf88f78bf489590c54dc9524bc6143 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..16539095f41a03adb3690f5506d528efb640c265 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.062194140625, + "l0_per_token_0.001": 32.062194140625, + "l0_per_token_0.01": 32.062194140625, + "l0_per_token_0.05": 32.062194140625, + "l0_per_token_0.1": 32.062194140625, + "l0_per_token_0.5": 32.062194140625, + "l0_per_token_1.0": 24.509033740234376, + "l1_per_token": 66.6345748946161, + "r2_score": 0.8862758749591823, + "l2_loss": 0.07571422307962093 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed2/config.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..82e88bb0886f7281a92deafc94ccfbf0c2d948b3 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..073588908150db25601ca809104a25810ce92d43 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.0752048828125, + "l0_per_token_0.001": 32.0752048828125, + "l0_per_token_0.01": 32.0752048828125, + "l0_per_token_0.05": 32.0752048828125, + "l0_per_token_0.1": 32.0752048828125, + "l0_per_token_0.5": 32.0752048828125, + "l0_per_token_1.0": 24.476112158203126, + "l1_per_token": 66.70951532040115, + "r2_score": 0.9234362244403416, + "l2_loss": 0.07537898424644615 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_64/code_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..a2df177761b95f2c7e87b53cbda2f4cf968f28fe --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 64.226845, + "recon_r2": 0.9102564008694842, + "l1_per_token": 92.72408584244768 + }, + "seed1": { + "l0": 64.52018833333334, + "recon_r2": 0.9098064712632679, + "l1_per_token": 93.05692981025467 + }, + "seed2": { + "l0": 64.36539333333333, + "recon_r2": 0.9102382427660436, + "l1_per_token": 92.9310809914731 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6384320875607069, + "mcc_abs": 0.6384949312118845, + "act_iou": 0.3042440304023016, + "act_iou_n_pairs": 3072, + "input_r2": 0.9457497650701744, + "matched_mcc_signed": { + "0.6": 0.7993678949461751, + "0.7": 0.8344894031762765, + "0.8": 0.8725565806001316, + "0.9": 0.9260450157692681 + }, + "matched_mcc_abs": { + "0.6": 0.7983075518466713, + "0.7": 0.8338311060975013, + "0.8": 0.871768300419215, + "0.9": 0.9261351622447189 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6384399932727974, + "mcc_abs": 0.6385094863739121, + "act_iou": 0.30624567421704085, + "act_iou_n_pairs": 3072, + "input_r2": 0.9461828266129311, + "matched_mcc_signed": { + "0.6": 0.7972212549349992, + "0.7": 0.831499172593444, + "0.8": 0.8728007137528464, + "0.9": 0.9256438194655625 + }, + "matched_mcc_abs": { + "0.6": 0.7971118147276791, + "0.7": 0.8309301009501788, + "0.8": 0.8731376478952274, + "0.9": 0.9263714316545028 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6401840483072617, + "mcc_abs": 0.6402360546682966, + "act_iou": 0.3061273271976903, + "act_iou_n_pairs": 3072, + "input_r2": 0.9450604138104018, + "matched_mcc_signed": { + "0.6": 0.7984976955463836, + "0.7": 0.8314204974225914, + "0.8": 0.8706359431678353, + "0.9": 0.9224729431177179 + }, + "matched_mcc_abs": { + "0.6": 0.797617864209955, + "0.7": 0.8311745203953517, + "0.8": 0.8694534337790433, + "0.9": 0.9211922314898305 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 64.37080888888887, + "l0_std": 0.14674663193497908, + "recon_r2_mean": 0.9101003716329319, + "recon_r2_std": 0.0002546870621893319, + "l1_per_token_mean": 92.90403221472515, + "l1_per_token_std": 0.16806250018682342, + "input_r2_mean": 0.9456643351645023, + "input_r2_std": 0.0005660621223926904, + "mcc_signed_mean": 0.6390187097135888, + "mcc_signed_std": 0.0010092205673185607, + "mcc_abs_mean": 0.639080157418031, + "mcc_abs_std": 0.0010010628367210215, + "matched_mcc_signed_per_threshold": { + "0.6": 0.7983622818091861, + "0.7": 0.832469691064104, + "0.8": 0.8719977458402711, + "0.9": 0.9247205927841828 + }, + "matched_mcc_signed_band": 0.856887577874436, + "matched_mcc_abs_per_threshold": { + "0.6": 0.7976790769281018, + "0.7": 0.831978575814344, + "0.8": 0.8714531273644952, + "0.9": 0.924566275129684 + }, + "matched_mcc_abs_band": 0.8564192638091562, + "act_iou_mean": 0.3055390106056776, + "act_iou_std": 0.0011230457691470564, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_64/dict_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..aa59c00b95feee2dd5f02e7cc311ccbb226f5dd2 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": 64, + "l0_raw": 64.06102353515625, + "recon_r2": 0.9105327462054292, + "l1_per_token": 92.48885242179031, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 130, + "dense_rate": 0.042317708333333336, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.1565461903810501, + "n_dup_abs": { + "0.5": 258, + "0.6": 222, + "0.7": 202, + "0.8": 176, + "0.9": 122 + }, + "n_dup_clusters_abs": { + "0.5": 129, + "0.6": 111, + "0.7": 101, + "0.8": 88, + "0.9": 61 + }, + "n_dup_removed_abs": { + "0.5": 129, + "0.6": 111, + "0.7": 101, + "0.8": 88, + "0.9": 61 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.27791574597358704, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": 64, + "l0_raw": 64.31955166015625, + "recon_r2": 0.9101955195313052, + "l1_per_token": 92.79874542392751, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 139, + "dense_rate": 0.045247395833333336, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.1568247526884079, + "n_dup_abs": { + "0.5": 268, + "0.6": 224, + "0.7": 202, + "0.8": 166, + "0.9": 116 + }, + "n_dup_clusters_abs": { + "0.5": 134, + "0.6": 112, + "0.7": 101, + "0.8": 83, + "0.9": 58 + }, + "n_dup_removed_abs": { + "0.5": 134, + "0.6": 112, + "0.7": 101, + "0.8": 83, + "0.9": 58 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.27696120738983154, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": 64, + "l0_raw": 64.17662749023438, + "recon_r2": 0.9395231925603862, + "l1_per_token": 92.67117433829893, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 139, + "dense_rate": 0.045247395833333336, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.15641331672668457, + "n_dup_abs": { + "0.5": 282, + "0.6": 230, + "0.7": 204, + "0.8": 192, + "0.9": 132 + }, + "n_dup_clusters_abs": { + "0.5": 141, + "0.6": 115, + "0.7": 102, + "0.8": 96, + "0.9": 66 + }, + "n_dup_removed_abs": { + "0.5": 141, + "0.6": 115, + "0.7": 102, + "0.8": 96, + "0.9": 66 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.27894127368927, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.6559428572654724, + "dcs_abs": 0.6596974730491638, + "frac_sign_flipped": 0.055013020833333336, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.6559428572654724, + "0.6": 0.6559428572654724, + "0.7": 0.6559428572654724, + "0.8": 0.6559428572654724, + "0.9": 0.6559428572654724 + }, + "unique_frac_signed": { + "0.5": 1.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6559428572654724, + "0.6": 0.6559428572654724, + "0.7": 0.6559428572654724, + "0.8": 0.6559428572654724, + "0.9": 0.6559428572654724 + }, + "matched_frac_signed": { + "0.5": 0.7561848958333334, + "0.6": 0.6705729166666666, + "0.7": 0.5367838541666666, + "0.8": 0.3577473958333333, + "0.9": 0.12141927083333333 + }, + "matched_cos_abs": { + "0.5": 0.6621581315994263, + "0.6": 0.6625770926475525, + "0.7": 0.6622603535652161, + "0.8": 0.6613072156906128, + "0.9": 0.6605029702186584 + }, + "unique_frac_abs": { + "0.5": 0.9571940104166666, + "0.6": 0.9637044270833334, + "0.7": 0.9671223958333334, + "0.8": 0.97216796875, + "0.9": 0.9806315104166666 + }, + "dedup_dcs_abs": { + "0.5": 0.6338137975156617, + "0.6": 0.6385284774684503, + "0.7": 0.6404868198054222, + "0.8": 0.6429016925976612, + "0.9": 0.6477100253202176 + }, + "matched_frac_abs": { + "0.5": 0.7314453125, + "0.6": 0.6572265625, + "0.7": 0.5276692708333334, + "0.8": 0.3502604166666667, + "0.9": 0.1181640625 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.6588645577430725, + "dcs_abs": 0.6625414490699768, + "frac_sign_flipped": 0.0556640625, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.6588645577430725, + "0.6": 0.6588645577430725, + "0.7": 0.6588645577430725, + "0.8": 0.6588645577430725, + "0.9": 0.6588645577430725 + }, + "unique_frac_signed": { + "0.5": 1.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6588645577430725, + "0.6": 0.6588645577430725, + "0.7": 0.6588645577430725, + "0.8": 0.6588645577430725, + "0.9": 0.6588645577430725 + }, + "matched_frac_signed": { + "0.5": 0.75390625, + "0.6": 0.6774088541666666, + "0.7": 0.5432942708333334, + "0.8": 0.3639322916666667, + "0.9": 0.13020833333333334 + }, + "matched_cos_abs": { + "0.5": 0.6671007871627808, + "0.6": 0.66591876745224, + "0.7": 0.6650668978691101, + "0.8": 0.6651901602745056, + "0.9": 0.663812518119812 + }, + "unique_frac_abs": { + "0.5": 0.9560546875, + "0.6": 0.9632161458333334, + "0.7": 0.9669596354166666, + "0.8": 0.9700520833333334, + "0.9": 0.9793294270833334 + }, + "dedup_dcs_abs": { + "0.5": 0.6377848346019164, + "0.6": 0.6414237086234305, + "0.7": 0.6430928450912081, + "0.8": 0.6452691007871181, + "0.9": 0.6500911330610204 + }, + "matched_frac_abs": { + "0.5": 0.7301432291666666, + "0.6": 0.6627604166666666, + "0.7": 0.5341796875, + "0.8": 0.3570963541666667, + "0.9": 0.12662760416666666 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.6582034230232239, + "dcs_abs": 0.6619676351547241, + "frac_sign_flipped": 0.060221354166666664, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.6582034826278687, + "0.6": 0.6582034826278687, + "0.7": 0.6582034826278687, + "0.8": 0.6582034826278687, + "0.9": 0.6582034826278687 + }, + "unique_frac_signed": { + "0.5": 1.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6582034826278687, + "0.6": 0.6582034826278687, + "0.7": 0.6582034826278687, + "0.8": 0.6582034826278687, + "0.9": 0.6582034826278687 + }, + "matched_frac_signed": { + "0.5": 0.7630208333333334, + "0.6": 0.67578125, + "0.7": 0.5452473958333334, + "0.8": 0.3606770833333333, + "0.9": 0.12109375 + }, + "matched_cos_abs": { + "0.5": 0.6658214330673218, + "0.6": 0.6649757623672485, + "0.7": 0.6649245619773865, + "0.8": 0.6652268767356873, + "0.9": 0.6635261178016663 + }, + "unique_frac_abs": { + "0.5": 0.9552408854166666, + "0.6": 0.9630533854166666, + "0.7": 0.9669596354166666, + "0.8": 0.9708658854166666, + "0.9": 0.9798177083333334 + }, + "dedup_dcs_abs": { + "0.5": 0.6360198552526223, + "0.6": 0.6404071591678075, + "0.7": 0.6429552120292403, + "0.8": 0.6458460806849567, + "0.9": 0.650134640163742 + }, + "matched_frac_abs": { + "0.5": 0.736328125, + "0.6": 0.6608072916666666, + "0.7": 0.5354817708333334, + "0.8": 0.3544921875, + "0.9": 0.1181640625 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 136, + "n_dense_std": 5.196152422706632, + "l0_mean": 64, + "l0_std": 0.1295044299072275, + "recon_r2_mean": 0.9200838194323735, + "recon_r2_std": 0.016835835327212327, + "l1_per_token_mean": 92.65292406133892, + "l1_per_token_std": 0.1557505141251213, + "babel_mu1_signed_mean": 0.15659475326538086, + "babel_mu1_signed_std": 0.0002099729814778349, + "n_dup_signed_per_threshold": { + "0.5": 0.0, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.0, + "n_dup_signed_band_std": 0.0, + "n_dup_clusters_signed_per_threshold": { + "0.5": 0.0, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.0, + "n_dup_clusters_signed_band_std": 0.0, + "largest_dup_signed_per_threshold": { + "0.5": 1.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.0, + "largest_dup_signed_band_std": 0.0, + "dcs_signed_mean": 0.657670279343923, + "dcs_signed_std": 0.001532078672028976, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.6576702992121378, + "0.6": 0.6576702992121378, + "0.7": 0.6576702992121378, + "0.8": 0.6576702992121378, + "0.9": 0.6576702992121378 + }, + "dedup_dcs_signed_band": 0.6576702992121378, + "dedup_dcs_signed_band_std": 0.001532089043205364, + "matched_cos_signed_per_threshold": { + "0.5": 0.6576702992121378, + "0.6": 0.6576702992121378, + "0.7": 0.6576702992121378, + "0.8": 0.6576702992121378, + "0.9": 0.6576702992121378 + }, + "matched_cos_signed_band": 0.6576702992121378, + "matched_cos_signed_band_std": 0.001532089043205364, + "unique_frac_signed_per_threshold": { + "0.5": 1.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 1.0, + "unique_frac_signed_band_std": 0.0, + "matched_frac_signed_per_threshold": { + "0.5": 0.7577039930555557, + "0.6": 0.674587673611111, + "0.7": 0.5417751736111112, + "0.8": 0.36078559027777773, + "0.9": 0.1242404513888889 + }, + "matched_frac_signed_band": 0.4253472222222222, + "matched_frac_signed_band_std": 0.003553188257776725, + "babel_mu1_abs_mean": 0.27793940901756287, + "babel_mu1_abs_std": 0.000990245218256896, + "n_dup_abs_per_threshold": { + "0.5": 269.3333333333333, + "0.6": 225.33333333333334, + "0.7": 202.66666666666666, + "0.8": 178.0, + "0.9": 123.33333333333333 + }, + "n_dup_abs_band": 182.33333333333334, + "n_dup_abs_band_std": 6.448514040717701, + "n_dup_clusters_abs_per_threshold": { + "0.5": 134.66666666666666, + "0.6": 112.66666666666667, + "0.7": 101.33333333333333, + "0.8": 89.0, + "0.9": 61.666666666666664 + }, + "n_dup_clusters_abs_band": 91.16666666666667, + "n_dup_clusters_abs_band_std": 3.2242570203588503, + "largest_dup_abs_per_threshold": { + "0.5": 2.0, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.0, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.661402185757955, + "dcs_abs_std": 0.0015039447020983895, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6358728291234002, + "0.6": 0.6401197817532295, + "0.7": 0.6421782923086236, + "0.8": 0.6446722913565787, + "0.9": 0.6493119328483267 + }, + "dedup_dcs_abs_band": 0.6440705745666896, + "dedup_dcs_abs_band_std": 0.0014424545558825996, + "matched_cos_abs_per_threshold": { + "0.5": 0.6650267839431763, + "0.6": 0.664490540822347, + "0.7": 0.6640839378039042, + "0.8": 0.6639080842336019, + "0.9": 0.6626138687133789 + }, + "matched_cos_abs_band": 0.663774107893308, + "matched_cos_abs_band_std": 0.0018368150415781612, + "unique_frac_abs_per_threshold": { + "0.5": 0.9561631944444443, + "0.6": 0.9633246527777778, + "0.7": 0.9670138888888888, + "0.8": 0.9710286458333334, + "0.9": 0.9799262152777778 + }, + "unique_frac_abs_band": 0.9703233506944444, + "unique_frac_abs_band_std": 0.0005247814160740164, + "matched_frac_abs_per_threshold": { + "0.5": 0.7326388888888888, + "0.6": 0.6602647569444443, + "0.7": 0.532443576388889, + "0.8": 0.35394965277777785, + "0.9": 0.12098524305555554 + }, + "matched_frac_abs_band": 0.4169108072916667, + "matched_frac_abs_band_std": 0.0034295747893400555, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 3072 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed0/config.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7802b1620e053d9a724ed6ee8b685a6f429fd36c --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..1a70c56212ab6800a7efc081c6ce342a1b8d9af4 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.06102353515625, + "l0_per_token_0.001": 64.06102353515625, + "l0_per_token_0.01": 64.06102353515625, + "l0_per_token_0.05": 64.06102353515625, + "l0_per_token_0.1": 64.06102353515625, + "l0_per_token_0.5": 64.06102353515625, + "l0_per_token_1.0": 28.615816162109375, + "l1_per_token": 92.48885242179031, + "r2_score": 0.9105327462054292, + "l2_loss": 0.059564701945975775 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed1/config.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f5667ba42ee29ab3985e17f4dc8dacd59a854842 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..5484915abeee4308726aa46c869f18e2ae7f9361 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.31955166015625, + "l0_per_token_0.001": 64.31955166015625, + "l0_per_token_0.01": 64.31955166015625, + "l0_per_token_0.05": 64.31955166015625, + "l0_per_token_0.1": 64.31955166015625, + "l0_per_token_0.5": 64.31955166015625, + "l0_per_token_1.0": 28.707490283203125, + "l1_per_token": 92.79874542392751, + "r2_score": 0.9101955195313052, + "l2_loss": 0.059789217681962874 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed2/config.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a4eedd7c928a1af9fa774274161a7fc3fff23423 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "192,192,384,768,1536", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..3cccb7f790f9f41232878d245bca75573c2c89be --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.17662749023438, + "l0_per_token_0.001": 64.17662749023438, + "l0_per_token_0.01": 64.17662749023438, + "l0_per_token_0.05": 64.17662749023438, + "l0_per_token_0.1": 64.17662749023438, + "l0_per_token_0.5": 64.17662749023438, + "l0_per_token_1.0": 28.684505615234375, + "l1_per_token": 92.67117433829893, + "r2_score": 0.9395231925603862, + "l2_loss": 0.05954095500050006 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_128/code_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..0bca4e65083477af4aa6ade61d999be5b27dbde8 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 128.449865, + "recon_r2": 0.9430403720322666, + "l1_per_token": 131.13849929510738 + }, + "seed1": { + "l0": 128.72969166666667, + "recon_r2": 0.9425129256377694, + "l1_per_token": 131.2439861404265 + }, + "seed2": { + "l0": 128.57379166666666, + "recon_r2": 0.9430805093462958, + "l1_per_token": 131.1976070294395 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6149350597869119, + "mcc_abs": 0.6149851871602507, + "act_iou": 0.28586963341200816, + "act_iou_n_pairs": 6144, + "input_r2": 0.9485277764074161, + "matched_mcc_signed": { + "0.6": 0.8209921048572025, + "0.7": 0.8512771411274579, + "0.8": 0.8778306250172314, + "0.9": 0.9272821265737913 + }, + "matched_mcc_abs": { + "0.6": 0.822762743437101, + "0.7": 0.8554278379467539, + "0.8": 0.8869357309010968, + "0.9": 0.9320613658051711 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6177506511123582, + "mcc_abs": 0.6178014769514383, + "act_iou": 0.2849005762853892, + "act_iou_n_pairs": 6144, + "input_r2": 0.9498716933394354, + "matched_mcc_signed": { + "0.6": 0.8163004663764054, + "0.7": 0.846799937650264, + "0.8": 0.8777794254400176, + "0.9": 0.9255439811256428 + }, + "matched_mcc_abs": { + "0.6": 0.8183497112503895, + "0.7": 0.8519421548367877, + "0.8": 0.8869556063459314, + "0.9": 0.9353472770714353 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6164609310699276, + "mcc_abs": 0.6165116212053813, + "act_iou": 0.2849929976878836, + "act_iou_n_pairs": 6144, + "input_r2": 0.9486021712024457, + "matched_mcc_signed": { + "0.6": 0.8194729286499954, + "0.7": 0.85074986133722, + "0.8": 0.881910594397517, + "0.9": 0.9350308417248668 + }, + "matched_mcc_abs": { + "0.6": 0.8223117320295878, + "0.7": 0.8560600253304604, + "0.8": 0.8905166352991764, + "0.9": 0.9389403802840123 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 128.58444944444443, + "l0_std": 0.1402174454724394, + "recon_r2_mean": 0.9428779356721106, + "recon_r2_std": 0.0003167443683346082, + "l1_per_token_mean": 131.19336415499114, + "l1_per_token_std": 0.052871259882222285, + "input_r2_mean": 0.9490005469830991, + "input_r2_std": 0.0007553513268674601, + "mcc_signed_mean": 0.6163822139897326, + "mcc_signed_std": 0.00140944524974116, + "mcc_abs_mean": 0.6164327617723568, + "mcc_abs_std": 0.001409800040650974, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8189218332945344, + "0.7": 0.8496089800383139, + "0.8": 0.879173548284922, + "0.9": 0.9292856498081004 + }, + "matched_mcc_signed_band": 0.8692475028564677, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8211413955723593, + "0.7": 0.8544766727046674, + "0.8": 0.8881359908487348, + "0.9": 0.9354496743868729 + }, + "matched_mcc_abs_band": 0.8748009333781586, + "act_iou_mean": 0.2852544024617603, + "act_iou_std": 0.0005348058250588817, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_128/dict_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..c25409ccb18db653acef52999ad7448ce778e14a --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 128, + "l0_raw": 128.17268393554687, + "recon_r2": 0.9432088390699579, + "l1_per_token": 130.83459625074047, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 354, + "dense_rate": 0.0576171875, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 1, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.1658192276954651, + "n_dup_abs": { + "0.5": 901, + "0.6": 834, + "0.7": 792, + "0.8": 764, + "0.9": 698 + }, + "n_dup_clusters_abs": { + "0.5": 450, + "0.6": 417, + "0.7": 396, + "0.8": 382, + "0.9": 349 + }, + "n_dup_removed_abs": { + "0.5": 451, + "0.6": 417, + "0.7": 396, + "0.8": 382, + "0.9": 349 + }, + "largest_dup_abs": { + "0.5": 3, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.327654629945755, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 128, + "l0_raw": 128.37267841796876, + "recon_r2": 0.9427296121127977, + "l1_per_token": 130.87055115449124, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 367, + "dense_rate": 0.059733072916666664, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 0, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 1, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.16524221003055573, + "n_dup_abs": { + "0.5": 866, + "0.6": 818, + "0.7": 790, + "0.8": 756, + "0.9": 708 + }, + "n_dup_clusters_abs": { + "0.5": 433, + "0.6": 409, + "0.7": 395, + "0.8": 378, + "0.9": 354 + }, + "n_dup_removed_abs": { + "0.5": 433, + "0.6": 409, + "0.7": 395, + "0.8": 378, + "0.9": 354 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.32530561089515686, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 128, + "l0_raw": 128.29437705078124, + "recon_r2": 0.9616373401321464, + "l1_per_token": 130.8808547429709, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 371, + "dense_rate": 0.060384114583333336, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 1, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.16488522291183472, + "n_dup_abs": { + "0.5": 862, + "0.6": 810, + "0.7": 764, + "0.8": 740, + "0.9": 674 + }, + "n_dup_clusters_abs": { + "0.5": 431, + "0.6": 405, + "0.7": 382, + "0.8": 370, + "0.9": 337 + }, + "n_dup_removed_abs": { + "0.5": 431, + "0.6": 405, + "0.7": 382, + "0.8": 370, + "0.9": 337 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.3232918679714203, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.5804238319396973, + "dcs_abs": 0.5851728916168213, + "frac_sign_flipped": 0.07763671875, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.5804839134216309, + "0.6": 0.5804238319396973, + "0.7": 0.5804238319396973, + "0.8": 0.5804238319396973, + "0.9": 0.5804238319396973 + }, + "unique_frac_signed": { + "0.5": 0.9999186197916666, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5804366735198224, + "0.6": 0.5804238319396973, + "0.7": 0.5804238319396973, + "0.8": 0.5804238319396973, + "0.9": 0.5804238319396973 + }, + "matched_frac_signed": { + "0.5": 0.6767578125, + "0.6": 0.55810546875, + "0.7": 0.3846028645833333, + "0.8": 0.17545572916666666, + "0.9": 0.042643229166666664 + }, + "matched_cos_abs": { + "0.5": 0.5933855175971985, + "0.6": 0.5932047963142395, + "0.7": 0.5923521518707275, + "0.8": 0.592110812664032, + "0.9": 0.5908145904541016 + }, + "unique_frac_abs": { + "0.5": 0.9280598958333334, + "0.6": 0.9327799479166666, + "0.7": 0.9356282552083334, + "0.8": 0.9381510416666666, + "0.9": 0.9427897135416666 + }, + "dedup_dcs_abs": { + "0.5": 0.5506973016502646, + "0.6": 0.5533295390099131, + "0.7": 0.5542214103237105, + "0.8": 0.555489375682858, + "0.9": 0.5570139184904594 + }, + "matched_frac_abs": { + "0.5": 0.64892578125, + "0.6": 0.5390625, + "0.7": 0.3678385416666667, + "0.8": 0.16357421875, + "0.9": 0.038899739583333336 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.5844010710716248, + "dcs_abs": 0.5881586670875549, + "frac_sign_flipped": 0.06819661458333333, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.5844403505325317, + "0.6": 0.5844010710716248, + "0.7": 0.5844010710716248, + "0.8": 0.5844010710716248, + "0.9": 0.5844010710716248 + }, + "unique_frac_signed": { + "0.5": 0.9998372395833334, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5843452267775623, + "0.6": 0.5844010710716248, + "0.7": 0.5844010710716248, + "0.8": 0.5844010710716248, + "0.9": 0.5844010710716248 + }, + "matched_frac_signed": { + "0.5": 0.6826171875, + "0.6": 0.5611979166666666, + "0.7": 0.3885091145833333, + "0.8": 0.17692057291666666, + "0.9": 0.045572916666666664 + }, + "matched_cos_abs": { + "0.5": 0.5962021946907043, + "0.6": 0.5958846211433411, + "0.7": 0.5954414010047913, + "0.8": 0.5949841141700745, + "0.9": 0.5934187173843384 + }, + "unique_frac_abs": { + "0.5": 0.92822265625, + "0.6": 0.93310546875, + "0.7": 0.9366861979166666, + "0.8": 0.9388020833333334, + "0.9": 0.9441731770833334 + }, + "dedup_dcs_abs": { + "0.5": 0.5534083848178852, + "0.6": 0.5560231987328734, + "0.7": 0.5577417419893511, + "0.8": 0.5585723259331038, + "0.9": 0.5602900357334875 + }, + "matched_frac_abs": { + "0.5": 0.6544596354166666, + "0.6": 0.54052734375, + "0.7": 0.37158203125, + "0.8": 0.16487630208333334, + "0.9": 0.040852864583333336 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.5805232524871826, + "dcs_abs": 0.5845509171485901, + "frac_sign_flipped": 0.07291666666666667, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.5806065201759338, + "0.6": 0.5805232524871826, + "0.7": 0.5805232524871826, + "0.8": 0.5805232524871826, + "0.9": 0.5805232524871826 + }, + "unique_frac_signed": { + "0.5": 0.9999186197916666, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.5805592702963622, + "0.6": 0.5805232524871826, + "0.7": 0.5805232524871826, + "0.8": 0.5805232524871826, + "0.9": 0.5805232524871826 + }, + "matched_frac_signed": { + "0.5": 0.6785481770833334, + "0.6": 0.5579427083333334, + "0.7": 0.3798828125, + "0.8": 0.17301432291666666, + "0.9": 0.04052734375 + }, + "matched_cos_abs": { + "0.5": 0.5928118824958801, + "0.6": 0.5926095843315125, + "0.7": 0.5921700596809387, + "0.8": 0.5916498899459839, + "0.9": 0.5910854935646057 + }, + "unique_frac_abs": { + "0.5": 0.9296875, + "0.6": 0.9337565104166666, + "0.7": 0.936767578125, + "0.8": 0.9391276041666666, + "0.9": 0.9437662760416666 + }, + "dedup_dcs_abs": { + "0.5": 0.5511297970078886, + "0.6": 0.5533530575048644, + "0.7": 0.5547257126454497, + "0.8": 0.5556347436504439, + "0.9": 0.5578465550837184 + }, + "matched_frac_abs": { + "0.5": 0.6534830729166666, + "0.6": 0.5397135416666666, + "0.7": 0.3640950520833333, + "0.8": 0.16194661458333334, + "0.9": 0.037272135416666664 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 364, + "n_dense_std": 8.888194417315589, + "l0_mean": 128, + "l0_std": 0.10077872741645212, + "recon_r2_mean": 0.9491919304383006, + "recon_r2_std": 0.010780704126116689, + "l1_per_token_mean": 130.86200071606754, + "l1_per_token_std": 0.024285685573736622, + "babel_mu1_signed_mean": 0.16531555354595184, + "babel_mu1_signed_std": 0.0004713021190267444, + "n_dup_signed_per_threshold": { + "0.5": 1.3333333333333333, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.0, + "n_dup_signed_band_std": 0.0, + "n_dup_clusters_signed_per_threshold": { + "0.5": 0.6666666666666666, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.0, + "n_dup_clusters_signed_band_std": 0.0, + "largest_dup_signed_per_threshold": { + "0.5": 1.6666666666666667, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.0, + "largest_dup_signed_band_std": 0.0, + "dcs_signed_mean": 0.5817827184995016, + "dcs_signed_std": 0.0022681046614461474, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5817803901979156, + "0.6": 0.5817827184995016, + "0.7": 0.5817827184995016, + "0.8": 0.5817827184995016, + "0.9": 0.5817827184995016 + }, + "dedup_dcs_signed_band": 0.5817827184995016, + "dedup_dcs_signed_band_std": 0.0022681046614461474, + "matched_cos_signed_per_threshold": { + "0.5": 0.5818435947100321, + "0.6": 0.5817827184995016, + "0.7": 0.5817827184995016, + "0.8": 0.5817827184995016, + "0.9": 0.5817827184995016 + }, + "matched_cos_signed_band": 0.5817827184995016, + "matched_cos_signed_band_std": 0.0022681046614461474, + "unique_frac_signed_per_threshold": { + "0.5": 0.9998914930555555, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 1.0, + "unique_frac_signed_band_std": 0.0, + "matched_frac_signed_per_threshold": { + "0.5": 0.6793077256944445, + "0.6": 0.55908203125, + "0.7": 0.38433159722222215, + "0.8": 0.17513020833333334, + "0.9": 0.04291449652777777 + }, + "matched_frac_signed_band": 0.29036458333333326, + "matched_frac_signed_band_std": 0.0026079785740498337, + "babel_mu1_abs_mean": 0.3254173696041107, + "babel_mu1_abs_std": 0.0021835270820270386, + "n_dup_abs_per_threshold": { + "0.5": 876.3333333333334, + "0.6": 820.6666666666666, + "0.7": 782.0, + "0.8": 753.3333333333334, + "0.9": 693.3333333333334 + }, + "n_dup_abs_band": 762.3333333333334, + "n_dup_abs_band_std": 13.428824718989125, + "n_dup_clusters_abs_per_threshold": { + "0.5": 438.0, + "0.6": 410.3333333333333, + "0.7": 391.0, + "0.8": 376.6666666666667, + "0.9": 346.6666666666667 + }, + "n_dup_clusters_abs_band": 381.1666666666667, + "n_dup_clusters_abs_band_std": 6.714412359494562, + "largest_dup_abs_per_threshold": { + "0.5": 2.3333333333333335, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.0, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.5859608252843221, + "dcs_abs_std": 0.0019286250293437473, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5517451611586796, + "0.6": 0.5542352650825503, + "0.7": 0.5555629549861704, + "0.8": 0.5565654817554685, + "0.9": 0.5583835031025551 + }, + "dedup_dcs_abs_band": 0.5561868012316861, + "dedup_dcs_abs_band_std": 0.0017164430795564448, + "matched_cos_abs_per_threshold": { + "0.5": 0.594133198261261, + "0.6": 0.593899667263031, + "0.7": 0.5933212041854858, + "0.8": 0.5929149389266968, + "0.9": 0.5917729338010153 + }, + "matched_cos_abs_band": 0.5929771860440572, + "matched_cos_abs_band_std": 0.0016974155644787388, + "unique_frac_abs_per_threshold": { + "0.5": 0.9286566840277778, + "0.6": 0.9332139756944443, + "0.7": 0.9363606770833334, + "0.8": 0.9386935763888888, + "0.9": 0.9435763888888888 + }, + "unique_frac_abs_band": 0.9379611545138888, + "unique_frac_abs_band_std": 0.0005464202766515858, + "matched_frac_abs_per_threshold": { + "0.5": 0.6522894965277777, + "0.6": 0.5397677951388888, + "0.7": 0.3678385416666667, + "0.8": 0.16346571180555558, + "0.9": 0.03900824652777778 + }, + "matched_frac_abs_band": 0.2775200737847222, + "matched_frac_abs_band_std": 0.0018576863442258597, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 6144 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed0/config.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..82407dded34aeb2c95a0024234ae0475abda7f80 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..6dddfc52bb93e525becbe229034a95f879b0e544 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.17268393554687, + "l0_per_token_0.001": 128.17268393554687, + "l0_per_token_0.01": 128.17268393554687, + "l0_per_token_0.05": 128.17268393554687, + "l0_per_token_0.1": 128.17268393554687, + "l0_per_token_0.5": 101.54064038085937, + "l0_per_token_1.0": 33.200520556640626, + "l1_per_token": 130.83459625074047, + "r2_score": 0.9432088390699579, + "l2_loss": 0.03780990731794636 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed1/config.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c7a4a9fdcf08106174cb74ff78d2526490d56eb4 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..ea4d34624b5844ba953144e1c629a018bd359b42 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.37267841796876, + "l0_per_token_0.001": 128.37267841796876, + "l0_per_token_0.01": 128.37267841796876, + "l0_per_token_0.05": 128.37267841796876, + "l0_per_token_0.1": 128.37267841796876, + "l0_per_token_0.5": 101.87232431640625, + "l0_per_token_1.0": 33.092476513671876, + "l1_per_token": 130.87055115449124, + "r2_score": 0.9427296121127977, + "l2_loss": 0.038128962722656395 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed2/config.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..299f0e255d08395b093c773a1cf7e7d84cb69433 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 128, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..dce44b845ed00bf06cdd5e29f6fd7f2b4b504c3b --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 128.29437705078124, + "l0_per_token_0.001": 128.29437705078124, + "l0_per_token_0.01": 128.29437705078124, + "l0_per_token_0.05": 128.29437705078124, + "l0_per_token_0.1": 128.29437705078124, + "l0_per_token_0.5": 101.60608896484375, + "l0_per_token_1.0": 33.28177309570312, + "l1_per_token": 130.8808547429709, + "r2_score": 0.9616373401321464, + "l2_loss": 0.037769014298118964 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_32/code_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..593d2a3029c94b8c25c27f9e470ad9be784e0ef1 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 32.100455, + "recon_r2": 0.8973584422243748, + "l1_per_token": 65.64800272539924 + }, + "seed1": { + "l0": 32.13416333333333, + "recon_r2": 0.8973165632051788, + "l1_per_token": 65.67332523760517 + }, + "seed2": { + "l0": 32.179741666666665, + "recon_r2": 0.8973900754605069, + "l1_per_token": 65.72910542543897 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6542569618884796, + "mcc_abs": 0.6542937971912538, + "act_iou": 0.34282919813579005, + "act_iou_n_pairs": 6144, + "input_r2": 0.9557466861027626, + "matched_mcc_signed": { + "0.6": 0.813482164705898, + "0.7": 0.8433813685811636, + "0.8": 0.8765653161837988, + "0.9": 0.9193411071971526 + }, + "matched_mcc_abs": { + "0.6": 0.8089693841998719, + "0.7": 0.8411609697379093, + "0.8": 0.8761429621532432, + "0.9": 0.9190637192633665 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.649038822188483, + "mcc_abs": 0.6490695041548549, + "act_iou": 0.33788822850274286, + "act_iou_n_pairs": 6144, + "input_r2": 0.9552515275425953, + "matched_mcc_signed": { + "0.6": 0.8106871797975951, + "0.7": 0.8422161157344215, + "0.8": 0.8765096699141139, + "0.9": 0.9167882437851526 + }, + "matched_mcc_abs": { + "0.6": 0.8066570846704811, + "0.7": 0.8388255338659892, + "0.8": 0.8754373056782975, + "0.9": 0.9165161394778218 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6523900535790389, + "mcc_abs": 0.6524240842900669, + "act_iou": 0.3417784268325186, + "act_iou_n_pairs": 6144, + "input_r2": 0.9559939699516622, + "matched_mcc_signed": { + "0.6": 0.8111839944061661, + "0.7": 0.8411293214535557, + "0.8": 0.8768544213109387, + "0.9": 0.916099196084163 + }, + "matched_mcc_abs": { + "0.6": 0.8068187781287699, + "0.7": 0.8397088514668014, + "0.8": 0.8759783729095741, + "0.9": 0.9156949301527416 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 32.138119999999994, + "l0_std": 0.039791145825562306, + "recon_r2_mean": 0.8973550269633535, + "recon_r2_std": 3.687493629467256e-05, + "l1_per_token_mean": 65.6834777961478, + "l1_per_token_std": 0.04149358772334471, + "input_r2_mean": 0.9556640611990067, + "input_r2_std": 0.0003780546769129389, + "mcc_signed_mean": 0.6518952792186671, + "mcc_signed_std": 0.0026440209403352263, + "mcc_abs_mean": 0.6519291285453918, + "mcc_abs_std": 0.0026470824173855496, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8117844463032197, + "0.7": 0.8422422685897136, + "0.8": 0.8766431358029506, + "0.9": 0.9174095156888228 + }, + "matched_mcc_signed_band": 0.8620198415961767, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8074817489997077, + "0.7": 0.8398984516902334, + "0.8": 0.8758528802470383, + "0.9": 0.9170915962979768 + }, + "matched_mcc_abs_band": 0.8600811693087391, + "act_iou_mean": 0.3408319511570172, + "act_iou_std": 0.002602913249017437, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_32/dict_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..8c285773de0a335cd81bff5e51018bde155374c2 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 32, + "l0_raw": 31.9968689453125, + "recon_r2": 0.8977604204580382, + "l1_per_token": 65.46837522055442, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 27, + "dense_rate": 0.00439453125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 4, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 4, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 1 + }, + "babel_mu1_signed": 0.20820806920528412, + "n_dup_abs": { + "0.5": 214, + "0.6": 116, + "0.7": 78, + "0.8": 52, + "0.9": 30 + }, + "n_dup_clusters_abs": { + "0.5": 107, + "0.6": 58, + "0.7": 39, + "0.8": 26, + "0.9": 15 + }, + "n_dup_removed_abs": { + "0.5": 107, + "0.6": 58, + "0.7": 39, + "0.8": 26, + "0.9": 15 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2690138518810272, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 32, + "l0_raw": 32.035225244140626, + "recon_r2": 0.8977083692646993, + "l1_per_token": 65.49239157570278, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 27, + "dense_rate": 0.00439453125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 18, + "0.6": 6, + "0.7": 2, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 9, + "0.6": 3, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 9, + "0.6": 3, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.20807015895843506, + "n_dup_abs": { + "0.5": 198, + "0.6": 122, + "0.7": 80, + "0.8": 52, + "0.9": 26 + }, + "n_dup_clusters_abs": { + "0.5": 99, + "0.6": 61, + "0.7": 40, + "0.8": 26, + "0.9": 13 + }, + "n_dup_removed_abs": { + "0.5": 99, + "0.6": 61, + "0.7": 40, + "0.8": 26, + "0.9": 13 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.2678748667240143, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 32, + "l0_raw": 32.08099345703125, + "recon_r2": 0.9308805269226289, + "l1_per_token": 65.54754172455095, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 30, + "dense_rate": 0.0048828125, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8, + "0.6": 2, + "0.7": 2, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 4, + "0.6": 1, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 4, + "0.6": 1, + "0.7": 1, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.20761603116989136, + "n_dup_abs": { + "0.5": 194, + "0.6": 124, + "0.7": 88, + "0.8": 50, + "0.9": 24 + }, + "n_dup_clusters_abs": { + "0.5": 97, + "0.6": 62, + "0.7": 44, + "0.8": 25, + "0.9": 12 + }, + "n_dup_removed_abs": { + "0.5": 97, + "0.6": 62, + "0.7": 44, + "0.8": 25, + "0.9": 12 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.26844096183776855, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.6850873827934265, + "dcs_abs": 0.6886026263237, + "frac_sign_flipped": 0.0537109375, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6848856210708618, + "0.6": 0.6851146221160889, + "0.7": 0.6850431561470032, + "0.8": 0.6850515007972717, + "0.9": 0.6850873827934265 + }, + "unique_frac_signed": { + "0.5": 0.9989420572916666, + "0.6": 0.9996744791666666, + "0.7": 0.9998372395833334, + "0.8": 0.9999186197916666, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6841610513220076, + "0.6": 0.6848916030333687, + "0.7": 0.684931658237474, + "0.8": 0.6849957511634177, + "0.9": 0.6850873827934265 + }, + "matched_frac_signed": { + "0.5": 0.78271484375, + "0.6": 0.7117513020833334, + "0.7": 0.603515625, + "0.8": 0.43701171875, + "0.9": 0.18929036458333334 + }, + "matched_cos_abs": { + "0.5": 0.6879509687423706, + "0.6": 0.6881654262542725, + "0.7": 0.6884019374847412, + "0.8": 0.6882526278495789, + "0.9": 0.6884579062461853 + }, + "unique_frac_abs": { + "0.5": 0.9832356770833334, + "0.6": 0.9903157552083334, + "0.7": 0.9935709635416666, + "0.8": 0.9957682291666666, + "0.9": 0.9977213541666666 + }, + "dedup_dcs_abs": { + "0.5": 0.6764179365515399, + "0.6": 0.6815010638092645, + "0.7": 0.6839761763306644, + "0.8": 0.6853401004530799, + "0.9": 0.686889154506692 + }, + "matched_frac_abs": { + "0.5": 0.7726236979166666, + "0.6": 0.7088216145833334, + "0.7": 0.6023763020833334, + "0.8": 0.4352213541666667, + "0.9": 0.18831380208333334 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.6788093447685242, + "dcs_abs": 0.6831965446472168, + "frac_sign_flipped": 0.0595703125, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6785646080970764, + "0.6": 0.6787641048431396, + "0.7": 0.6787641048431396, + "0.8": 0.6787869334220886, + "0.9": 0.6788093447685242 + }, + "unique_frac_signed": { + "0.5": 0.9993489583333334, + "0.6": 0.9998372395833334, + "0.7": 0.9998372395833334, + "0.8": 0.9999186197916666, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6781228342636799, + "0.6": 0.678653628914617, + "0.7": 0.678653628914617, + "0.8": 0.6787316936000327, + "0.9": 0.6788093447685242 + }, + "matched_frac_signed": { + "0.5": 0.7737630208333334, + "0.6": 0.7052408854166666, + "0.7": 0.5916341145833334, + "0.8": 0.4280598958333333, + "0.9": 0.1787109375 + }, + "matched_cos_abs": { + "0.5": 0.6831341981887817, + "0.6": 0.6830999851226807, + "0.7": 0.6830841898918152, + "0.8": 0.6829200983047485, + "0.9": 0.6830306649208069 + }, + "unique_frac_abs": { + "0.5": 0.9833984375, + "0.6": 0.990234375, + "0.7": 0.9932454427083334, + "0.8": 0.995849609375, + "0.9": 0.997802734375 + }, + "dedup_dcs_abs": { + "0.5": 0.6717931031016633, + "0.6": 0.676429086830467, + "0.7": 0.6784702585961593, + "0.8": 0.6800857131311204, + "0.9": 0.6815298651199555 + }, + "matched_frac_abs": { + "0.5": 0.7644856770833334, + "0.6": 0.7019856770833334, + "0.7": 0.5909830729166666, + "0.8": 0.4267578125, + "0.9": 0.177734375 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.6838262677192688, + "dcs_abs": 0.687550961971283, + "frac_sign_flipped": 0.05615234375, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6839963793754578, + "0.6": 0.6839173436164856, + "0.7": 0.6837824583053589, + "0.8": 0.6838262677192688, + "0.9": 0.6838262677192688 + }, + "unique_frac_signed": { + "0.5": 0.9989420572916666, + "0.6": 0.9996744791666666, + "0.7": 0.9998372395833334, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6832727503933711, + "0.6": 0.6836947142728604, + "0.7": 0.6836711655875358, + "0.8": 0.6838262677192688, + "0.9": 0.6838262677192688 + }, + "matched_frac_signed": { + "0.5": 0.7775065104166666, + "0.6": 0.7052408854166666, + "0.7": 0.6004231770833334, + "0.8": 0.4363606770833333, + "0.9": 0.19319661458333334 + }, + "matched_cos_abs": { + "0.5": 0.6872850060462952, + "0.6": 0.6874541640281677, + "0.7": 0.6877214908599854, + "0.8": 0.6873652935028076, + "0.9": 0.6872654557228088 + }, + "unique_frac_abs": { + "0.5": 0.9840494791666666, + "0.6": 0.989990234375, + "0.7": 0.9931640625, + "0.8": 0.995849609375, + "0.9": 0.9979654947916666 + }, + "dedup_dcs_abs": { + "0.5": 0.6763224522389161, + "0.6": 0.6805729089683155, + "0.7": 0.6830202697310597, + "0.8": 0.6845124590327032, + "0.9": 0.6858672105736332 + }, + "matched_frac_abs": { + "0.5": 0.7692057291666666, + "0.6": 0.7019856770833334, + "0.7": 0.5983072916666666, + "0.8": 0.4348958333333333, + "0.9": 0.19205729166666666 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 28, + "n_dense_std": 1.7320508075688772, + "l0_mean": 32, + "l0_std": 0.042116640501729724, + "recon_r2_mean": 0.9087831055484554, + "recon_r2_std": 0.019136945965135474, + "l1_per_token_mean": 65.50276950693605, + "l1_per_token_std": 0.0405907616492977, + "babel_mu1_signed_mean": 0.20796475311120352, + "babel_mu1_signed_std": 0.00030977419729680565, + "n_dup_signed_per_threshold": { + "0.5": 11.333333333333334, + "0.6": 3.3333333333333335, + "0.7": 2.0, + "0.8": 0.6666666666666666, + "0.9": 0.0 + }, + "n_dup_signed_band": 1.5000000000000002, + "n_dup_signed_band_std": 0.5, + "n_dup_clusters_signed_per_threshold": { + "0.5": 5.666666666666667, + "0.6": 1.6666666666666667, + "0.7": 1.0, + "0.8": 0.3333333333333333, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.7500000000000001, + "n_dup_clusters_signed_band_std": 0.25, + "largest_dup_signed_per_threshold": { + "0.5": 2.0, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 1.3333333333333333, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.5833333333333333, + "largest_dup_signed_band_std": 0.14433756729740646, + "dcs_signed_mean": 0.6825743317604065, + "dcs_signed_std": 0.0033209860123688097, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.6818522119930196, + "0.6": 0.6824133154069486, + "0.7": 0.6824188175798757, + "0.8": 0.6825179041609064, + "0.9": 0.6825743317604065 + }, + "dedup_dcs_signed_band": 0.6824810922270343, + "dedup_dcs_signed_band_std": 0.0033207591679248897, + "matched_cos_signed_per_threshold": { + "0.5": 0.6824822028477987, + "0.6": 0.6825986901919047, + "0.7": 0.6825299064318339, + "0.8": 0.6825549006462097, + "0.9": 0.6825743317604065 + }, + "matched_cos_signed_band": 0.6825644572575886, + "matched_cos_signed_band_std": 0.0033342455764417828, + "unique_frac_signed_per_threshold": { + "0.5": 0.9990776909722222, + "0.6": 0.9997287326388888, + "0.7": 0.9998372395833334, + "0.8": 0.9999457465277777, + "0.9": 1.0 + }, + "unique_frac_signed_band": 0.9998779296875, + "unique_frac_signed_band_std": 2.034505208337034e-05, + "matched_frac_signed_per_threshold": { + "0.5": 0.7779947916666666, + "0.6": 0.7074110243055555, + "0.7": 0.5985243055555557, + "0.8": 0.43381076388888884, + "0.9": 0.18706597222222224 + }, + "matched_frac_signed_band": 0.4817030164930556, + "matched_frac_signed_band_std": 0.005078009797009449, + "babel_mu1_abs_mean": 0.26844322681427, + "babel_mu1_abs_std": 0.0005694959565816616, + "n_dup_abs_per_threshold": { + "0.5": 202.0, + "0.6": 120.66666666666667, + "0.7": 82.0, + "0.8": 51.333333333333336, + "0.9": 26.666666666666668 + }, + "n_dup_abs_band": 70.16666666666667, + "n_dup_abs_band_std": 1.2583057392117918, + "n_dup_clusters_abs_per_threshold": { + "0.5": 101.0, + "0.6": 60.333333333333336, + "0.7": 41.0, + "0.8": 25.666666666666668, + "0.9": 13.333333333333334 + }, + "n_dup_clusters_abs_band": 35.083333333333336, + "n_dup_clusters_abs_band_std": 0.6291528696058959, + "largest_dup_abs_per_threshold": { + "0.5": 2.0, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.0, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.6864500443140665, + "dcs_abs_std": 0.002866259677555376, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6748444972973732, + "0.6": 0.679501019869349, + "0.7": 0.6818222348859612, + "0.8": 0.6833127575389678, + "0.9": 0.6847620767334268 + }, + "dedup_dcs_abs_band": 0.6823495222569262, + "dedup_dcs_abs_band_std": 0.0028280624072874174, + "matched_cos_abs_per_threshold": { + "0.5": 0.6861233909924825, + "0.6": 0.6862398584683737, + "0.7": 0.6864025394121805, + "0.8": 0.6861793398857117, + "0.9": 0.6862513422966003 + }, + "matched_cos_abs_band": 0.6862682700157166, + "matched_cos_abs_band_std": 0.0028346015210938915, + "unique_frac_abs_per_threshold": { + "0.5": 0.9835611979166666, + "0.6": 0.9901801215277778, + "0.7": 0.9933268229166666, + "0.8": 0.9958224826388888, + "0.9": 0.997829861111111 + }, + "unique_frac_abs_band": 0.994289822048611, + "unique_frac_abs_band_std": 5.120059160201779e-05, + "matched_frac_abs_per_threshold": { + "0.5": 0.7687717013888888, + "0.6": 0.7042643229166666, + "0.7": 0.5972222222222222, + "0.8": 0.4322916666666667, + "0.9": 0.18603515625 + }, + "matched_frac_abs_band": 0.4799533420138889, + "matched_frac_abs_band_std": 0.004929104115266704, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 6144 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed0/config.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4799215e562b7b652f2a6388902ec651236cf521 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..d1cc2f6cdb5a86c190586a56c72f0017dfe9e4e2 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.9968689453125, + "l0_per_token_0.001": 31.9968689453125, + "l0_per_token_0.01": 31.9968689453125, + "l0_per_token_0.05": 31.9968689453125, + "l0_per_token_0.1": 31.9968689453125, + "l0_per_token_0.5": 31.9968689453125, + "l0_per_token_1.0": 23.483521142578127, + "l1_per_token": 65.46837522055442, + "r2_score": 0.8977604204580382, + "l2_loss": 0.06806814587694877 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed1/config.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..77133a8368ff520168fe29f6cc0b96c43ebb64da --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..41c2ff3cb9b5cc631a9bbf84d6f97737f0a960d3 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.035225244140626, + "l0_per_token_0.001": 32.035225244140626, + "l0_per_token_0.01": 32.035225244140626, + "l0_per_token_0.05": 32.035225244140626, + "l0_per_token_0.1": 32.035225244140626, + "l0_per_token_0.5": 32.035225244140626, + "l0_per_token_1.0": 23.520864013671876, + "l1_per_token": 65.49239157570278, + "r2_score": 0.8977083692646993, + "l2_loss": 0.06810280005136081 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed2/config.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..52e803cf6d9f59ccf0bec19d110903bdbe209e58 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 32, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..7acce263e5c16fbea869c6a5475c810e2edf4e84 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.08099345703125, + "l0_per_token_0.001": 32.08099345703125, + "l0_per_token_0.01": 32.08099345703125, + "l0_per_token_0.05": 32.08099345703125, + "l0_per_token_0.1": 32.08099345703125, + "l0_per_token_0.5": 32.08099345703125, + "l0_per_token_1.0": 23.503432666015627, + "l1_per_token": 65.54754172455095, + "r2_score": 0.9308805269226289, + "l2_loss": 0.06804987912543659 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_64/code_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..103959e1d51e8475acb596ba8fad4439dfbfe91b --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 64.24707333333333, + "recon_r2": 0.919919510206296, + "l1_per_token": 89.7169276366369 + }, + "seed1": { + "l0": 64.451985, + "recon_r2": 0.9196296513036955, + "l1_per_token": 89.98946997237037 + }, + "seed2": { + "l0": 64.30788666666666, + "recon_r2": 0.9198841244774912, + "l1_per_token": 89.63258802214573 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.6338481241926994, + "mcc_abs": 0.6338882168526188, + "act_iou": 0.2902694134193776, + "act_iou_n_pairs": 6144, + "input_r2": 0.9505153976764962, + "matched_mcc_signed": { + "0.6": 0.8045402895108085, + "0.7": 0.8381831606987656, + "0.8": 0.8768431711925844, + "0.9": 0.9225763514437242 + }, + "matched_mcc_abs": { + "0.6": 0.803498853201192, + "0.7": 0.837812426815197, + "0.8": 0.8766992863832619, + "0.9": 0.9216045781481002 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6301769364523346, + "mcc_abs": 0.6302205060854772, + "act_iou": 0.28510170669082174, + "act_iou_n_pairs": 6144, + "input_r2": 0.9507429669888289, + "matched_mcc_signed": { + "0.6": 0.8008958199171469, + "0.7": 0.8355988164569065, + "0.8": 0.8759532924590832, + "0.9": 0.9261909641273509 + }, + "matched_mcc_abs": { + "0.6": 0.798827387014593, + "0.7": 0.8347236461639468, + "0.8": 0.8765170048166147, + "0.9": 0.9247198810766272 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.6320629559651232, + "mcc_abs": 0.6321009286819194, + "act_iou": 0.2872313959066089, + "act_iou_n_pairs": 6144, + "input_r2": 0.9509301238487923, + "matched_mcc_signed": { + "0.6": 0.8036323192938393, + "0.7": 0.8385465258095054, + "0.8": 0.87684206803913, + "0.9": 0.9240160898360475 + }, + "matched_mcc_abs": { + "0.6": 0.8012797470058213, + "0.7": 0.8369208511903022, + "0.8": 0.8774377121896911, + "0.9": 0.9235245648613806 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 64.33564833333332, + "l0_std": 0.10523892049099773, + "recon_r2_mean": 0.9198110953291608, + "recon_r2_std": 0.00015812807541228582, + "l1_per_token_mean": 89.779661877051, + "l1_per_token_std": 0.18652847045233495, + "input_r2_mean": 0.9507294961713724, + "input_r2_std": 0.00020769098846375506, + "mcc_signed_mean": 0.6320293388700524, + "mcc_signed_std": 0.0018358247296683663, + "mcc_abs_mean": 0.6320698838733384, + "mcc_abs_std": 0.0018340524537091103, + "matched_mcc_signed_per_threshold": { + "0.6": 0.8030228095739315, + "0.7": 0.8374428343217258, + "0.8": 0.8765461772302658, + "0.9": 0.9242611351357075 + }, + "matched_mcc_signed_band": 0.8603182390654076, + "matched_mcc_abs_per_threshold": { + "0.6": 0.8012019957405355, + "0.7": 0.8364856413898153, + "0.8": 0.8768846677965225, + "0.9": 0.9232830080287027 + }, + "matched_mcc_abs_band": 0.859463828238894, + "act_iou_mean": 0.2875341720056028, + "act_iou_std": 0.002597124031054711, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_64/dict_metrics_step_final.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..0a7473d183b5eabe38898f02e60d8c8ead61c1a9 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 64.07642202148438, + "recon_r2": 0.9202243028885576, + "l1_per_token": 89.47779828809344, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 133, + "dense_rate": 0.021647135416666668, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.17722387611865997, + "n_dup_abs": { + "0.5": 346, + "0.6": 268, + "0.7": 240, + "0.8": 192, + "0.9": 150 + }, + "n_dup_clusters_abs": { + "0.5": 173, + "0.6": 134, + "0.7": 120, + "0.8": 96, + "0.9": 75 + }, + "n_dup_removed_abs": { + "0.5": 173, + "0.6": 134, + "0.7": 120, + "0.8": 96, + "0.9": 75 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.26757511496543884, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 64.26222543945312, + "recon_r2": 0.9199588169092011, + "l1_per_token": 89.72088045650828, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 136, + "dense_rate": 0.022135416666666668, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.17613525688648224, + "n_dup_abs": { + "0.5": 360, + "0.6": 278, + "0.7": 244, + "0.8": 210, + "0.9": 158 + }, + "n_dup_clusters_abs": { + "0.5": 180, + "0.6": 139, + "0.7": 122, + "0.8": 105, + "0.9": 79 + }, + "n_dup_removed_abs": { + "0.5": 180, + "0.6": 139, + "0.7": 122, + "0.8": 105, + "0.9": 79 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.26604732871055603, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": 64, + "l0_raw": 64.10941928710938, + "recon_r2": 0.9460386593955417, + "l1_per_token": 89.37291527930226, + "stats_source": "stats", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 139, + "dense_rate": 0.022623697916666668, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 2, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_clusters_signed": { + "0.5": 1, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "n_dup_removed_signed": { + "0.5": 1, + "0.6": 0, + "0.7": 0, + "0.8": 0, + "0.9": 0 + }, + "largest_dup_signed": { + "0.5": 2, + "0.6": 1, + "0.7": 1, + "0.8": 1, + "0.9": 1 + }, + "babel_mu1_signed": 0.17617511749267578, + "n_dup_abs": { + "0.5": 358, + "0.6": 274, + "0.7": 234, + "0.8": 210, + "0.9": 152 + }, + "n_dup_clusters_abs": { + "0.5": 179, + "0.6": 137, + "0.7": 117, + "0.8": 105, + "0.9": 76 + }, + "n_dup_removed_abs": { + "0.5": 179, + "0.6": 137, + "0.7": 117, + "0.8": 105, + "0.9": 76 + }, + "largest_dup_abs": { + "0.5": 2, + "0.6": 2, + "0.7": 2, + "0.8": 2, + "0.9": 2 + }, + "babel_mu1_abs": 0.268228679895401, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.6393224000930786, + "dcs_abs": 0.6434466242790222, + "frac_sign_flipped": 0.059407552083333336, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6393988132476807, + "0.6": 0.6393224000930786, + "0.7": 0.6393224000930786, + "0.8": 0.6393224000930786, + "0.9": 0.6393224000930786 + }, + "unique_frac_signed": { + "0.5": 0.9996744791666666, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6391906756131599, + "0.6": 0.6393224000930786, + "0.7": 0.6393224000930786, + "0.8": 0.6393224000930786, + "0.9": 0.6393224000930786 + }, + "matched_frac_signed": { + "0.5": 0.7415364583333334, + "0.6": 0.6461588541666666, + "0.7": 0.5055338541666666, + "0.8": 0.31298828125, + "0.9": 0.09163411458333333 + }, + "matched_cos_abs": { + "0.5": 0.6452811360359192, + "0.6": 0.6451159119606018, + "0.7": 0.6447437405586243, + "0.8": 0.644852340221405, + "0.9": 0.6440861821174622 + }, + "unique_frac_abs": { + "0.5": 0.9712727864583334, + "0.6": 0.977783203125, + "0.7": 0.9803059895833334, + "0.8": 0.983642578125, + "0.9": 0.9874674479166666 + }, + "dedup_dcs_abs": { + "0.5": 0.6267440070466062, + "0.6": 0.6307835027837427, + "0.7": 0.6320461506159821, + "0.8": 0.6343042184453225, + "0.9": 0.6360141384939197 + }, + "matched_frac_abs": { + "0.5": 0.7298177083333334, + "0.6": 0.6375325520833334, + "0.7": 0.5003255208333334, + "0.8": 0.3095703125, + "0.9": 0.09016927083333333 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.6322768330574036, + "dcs_abs": 0.6367434859275818, + "frac_sign_flipped": 0.061686197916666664, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6323253512382507, + "0.6": 0.6322768330574036, + "0.7": 0.6322768330574036, + "0.8": 0.6322768330574036, + "0.9": 0.6322768330574036 + }, + "unique_frac_signed": { + "0.5": 0.999755859375, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6321709749317961, + "0.6": 0.6322768330574036, + "0.7": 0.6322768330574036, + "0.8": 0.6322768330574036, + "0.9": 0.6322768330574036 + }, + "matched_frac_signed": { + "0.5": 0.740234375, + "0.6": 0.638671875, + "0.7": 0.4912109375, + "0.8": 0.2911783854166667, + "0.9": 0.08479817708333333 + }, + "matched_cos_abs": { + "0.5": 0.6376492381095886, + "0.6": 0.6378955841064453, + "0.7": 0.6376672387123108, + "0.8": 0.6377464532852173, + "0.9": 0.6368857026100159 + }, + "unique_frac_abs": { + "0.5": 0.9713541666666666, + "0.6": 0.9779459635416666, + "0.7": 0.980712890625, + "0.8": 0.983642578125, + "0.9": 0.9877115885416666 + }, + "dedup_dcs_abs": { + "0.5": 0.6193832443095744, + "0.6": 0.6238274116379519, + "0.7": 0.6253684809344122, + "0.8": 0.627314565499546, + "0.9": 0.6290593890444143 + }, + "matched_frac_abs": { + "0.5": 0.7265625, + "0.6": 0.63134765625, + "0.7": 0.48583984375, + "0.8": 0.2869466145833333, + "0.9": 0.08284505208333333 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.6346152424812317, + "dcs_abs": 0.6385049223899841, + "frac_sign_flipped": 0.059895833333333336, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.6346324682235718, + "0.6": 0.6346152424812317, + "0.7": 0.6346152424812317, + "0.8": 0.6346152424812317, + "0.9": 0.6346152424812317 + }, + "unique_frac_signed": { + "0.5": 0.999755859375, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "dedup_dcs_signed": { + "0.5": 0.6344775286561344, + "0.6": 0.6346152424812317, + "0.7": 0.6346152424812317, + "0.8": 0.6346152424812317, + "0.9": 0.6346152424812317 + }, + "matched_frac_signed": { + "0.5": 0.7418619791666666, + "0.6": 0.640625, + "0.7": 0.4910481770833333, + "0.8": 0.2976888020833333, + "0.9": 0.09163411458333333 + }, + "matched_cos_abs": { + "0.5": 0.6393865346908569, + "0.6": 0.639886200428009, + "0.7": 0.6399271488189697, + "0.8": 0.6393829584121704, + "0.9": 0.6390058994293213 + }, + "unique_frac_abs": { + "0.5": 0.9707845052083334, + "0.6": 0.9775390625, + "0.7": 0.9805501302083334, + "0.8": 0.98291015625, + "0.9": 0.9873860677083334 + }, + "dedup_dcs_abs": { + "0.5": 0.6207065407167345, + "0.6": 0.625513756473083, + "0.7": 0.6274806490982883, + "0.8": 0.6284560035564937, + "0.9": 0.6309455222799443 + }, + "matched_frac_abs": { + "0.5": 0.7286783854166666, + "0.6": 0.6331380208333334, + "0.7": 0.4860026041666667, + "0.8": 0.29296875, + "0.9": 0.08984375 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 136, + "n_dense_std": 3.0, + "l0_mean": 64, + "l0_std": 0.09913076369262054, + "recon_r2_mean": 0.9287405930644335, + "recon_r2_std": 0.014981152986264875, + "l1_per_token_mean": 89.52386467463465, + "l1_per_token_std": 0.17849796949469302, + "babel_mu1_signed_mean": 0.176511416832606, + "babel_mu1_signed_std": 0.000617329646700152, + "n_dup_signed_per_threshold": { + "0.5": 3.3333333333333335, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_signed_band": 0.0, + "n_dup_signed_band_std": 0.0, + "n_dup_clusters_signed_per_threshold": { + "0.5": 1.6666666666666667, + "0.6": 0.0, + "0.7": 0.0, + "0.8": 0.0, + "0.9": 0.0 + }, + "n_dup_clusters_signed_band": 0.0, + "n_dup_clusters_signed_band_std": 0.0, + "largest_dup_signed_per_threshold": { + "0.5": 2.0, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "largest_dup_signed_band": 1.0, + "largest_dup_signed_band_std": 0.0, + "dcs_signed_mean": 0.6354048252105713, + "dcs_signed_std": 0.0035885351298825616, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.6352797264003635, + "0.6": 0.6354048252105713, + "0.7": 0.6354048252105713, + "0.8": 0.6354048252105713, + "0.9": 0.6354048252105713 + }, + "dedup_dcs_signed_band": 0.6354048252105713, + "dedup_dcs_signed_band_std": 0.0035885351298825616, + "matched_cos_signed_per_threshold": { + "0.5": 0.6354522109031677, + "0.6": 0.6354048252105713, + "0.7": 0.6354048252105713, + "0.8": 0.6354048252105713, + "0.9": 0.6354048252105713 + }, + "matched_cos_signed_band": 0.6354048252105713, + "matched_cos_signed_band_std": 0.0035885351298825616, + "unique_frac_signed_per_threshold": { + "0.5": 0.9997287326388888, + "0.6": 1.0, + "0.7": 1.0, + "0.8": 1.0, + "0.9": 1.0 + }, + "unique_frac_signed_band": 1.0, + "unique_frac_signed_band_std": 0.0, + "matched_frac_signed_per_threshold": { + "0.5": 0.7412109375, + "0.6": 0.6418185763888888, + "0.7": 0.4959309895833333, + "0.8": 0.3006184895833333, + "0.9": 0.08935546875 + }, + "matched_frac_signed_band": 0.38193088107638884, + "matched_frac_signed_band_std": 0.006472967310500207, + "babel_mu1_abs_mean": 0.26728370785713196, + "babel_mu1_abs_std": 0.0011194917708429012, + "n_dup_abs_per_threshold": { + "0.5": 354.6666666666667, + "0.6": 273.3333333333333, + "0.7": 239.33333333333334, + "0.8": 204.0, + "0.9": 153.33333333333334 + }, + "n_dup_abs_band": 217.5, + "n_dup_abs_band_std": 5.0, + "n_dup_clusters_abs_per_threshold": { + "0.5": 177.33333333333334, + "0.6": 136.66666666666666, + "0.7": 119.66666666666667, + "0.8": 102.0, + "0.9": 76.66666666666667 + }, + "n_dup_clusters_abs_band": 108.75, + "n_dup_clusters_abs_band_std": 2.5, + "largest_dup_abs_per_threshold": { + "0.5": 2.0, + "0.6": 2.0, + "0.7": 2.0, + "0.8": 2.0, + "0.9": 2.0 + }, + "largest_dup_abs_band": 2.0, + "largest_dup_abs_band_std": 0.0, + "dcs_abs_mean": 0.6395650108655294, + "dcs_abs_std": 0.0034750333266928197, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.6222779306909717, + "0.6": 0.6267082236315925, + "0.7": 0.6282984268828943, + "0.8": 0.6300249291671207, + "0.9": 0.6320063499394261 + }, + "dedup_dcs_abs_band": 0.6292594824052584, + "dedup_dcs_abs_band_std": 0.003590785797029396, + "matched_cos_abs_per_threshold": { + "0.5": 0.640772302945455, + "0.6": 0.6409658988316854, + "0.7": 0.6407793760299683, + "0.8": 0.6406605839729309, + "0.9": 0.6399925947189331 + }, + "matched_cos_abs_band": 0.6405996133883795, + "matched_cos_abs_band_std": 0.003689021471805628, + "unique_frac_abs_per_threshold": { + "0.5": 0.9711371527777778, + "0.6": 0.9777560763888888, + "0.7": 0.9805230034722223, + "0.8": 0.9833984375, + "0.9": 0.9875217013888888 + }, + "unique_frac_abs_band": 0.9822998046875, + "unique_frac_abs_band_std": 0.00020345052083325932, + "matched_frac_abs_per_threshold": { + "0.5": 0.7283528645833334, + "0.6": 0.634006076388889, + "0.7": 0.49072265625, + "0.8": 0.2964952256944444, + "0.9": 0.08761935763888888 + }, + "matched_frac_abs_band": 0.3772108289930555, + "matched_frac_abs_band_std": 0.006500788032278178, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 6144 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed0/config.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8fcb63e0a3fa994dbbcc69d24b9250bcc4ed6262 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed0/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..52ca46030e78ec47f67ba20cf04102c1e2aadf45 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.07642202148438, + "l0_per_token_0.001": 64.07642202148438, + "l0_per_token_0.01": 64.07642202148438, + "l0_per_token_0.05": 64.07642202148438, + "l0_per_token_0.1": 64.07642202148438, + "l0_per_token_0.5": 64.07642202148438, + "l0_per_token_1.0": 27.018734130859375, + "l1_per_token": 89.47779828809344, + "r2_score": 0.9202243028885576, + "l2_loss": 0.0531123446784936 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed1/config.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b51a95f2c49d8264c95d2df553e8f47969767674 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed1/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..1c685c61487881011bf0a5c395765dea1f799d79 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.26222543945312, + "l0_per_token_0.001": 64.26222543945312, + "l0_per_token_0.01": 64.26222543945312, + "l0_per_token_0.05": 64.26222543945312, + "l0_per_token_0.1": 64.26222543945312, + "l0_per_token_0.5": 64.26222543945312, + "l0_per_token_1.0": 27.032914208984376, + "l1_per_token": 89.72088045650828, + "r2_score": 0.9199588169092011, + "l2_loss": 0.05328909754125073 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed2/config.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d79b55bff820762f7c49c954267bbf9d1b1911c4 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed2/config.json @@ -0,0 +1,13 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "matryoshka", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "top_k": 64, + "top_k_inference": 0, + "group_sizes_csv": "384,384,768,1536,3072", + "class_type": "MatryoshkaBatchTopKSAE", + "module_name": "consdl.models.batchtopk" +} diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..4f20309da08a1856c40f6d9d30c2934ee66c8977 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 64.10941928710938, + "l0_per_token_0.001": 64.10941928710938, + "l0_per_token_0.01": 64.10941928710938, + "l0_per_token_0.05": 64.10941928710938, + "l0_per_token_0.1": 64.10941928710938, + "l0_per_token_0.5": 64.10941928710938, + "l0_per_token_1.0": 26.984386474609376, + "l1_per_token": 89.37291527930226, + "r2_score": 0.9460386593955417, + "l2_loss": 0.05312631219670122 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/matryoshka/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/code_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x16/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..7c0019777f9951bcdedc141673456017af0c1340 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 355.9318166666667, + "recon_r2": 0.9834826408282966, + "l1_per_token": 156.23972150447466 + }, + "seed1": { + "l0": 360.54515166666664, + "recon_r2": 0.9838089909407193, + "l1_per_token": 156.77521415513394 + }, + "seed2": { + "l0": 341.67803833333335, + "recon_r2": 0.9827451662271783, + "l1_per_token": 153.6285543277214 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.39930654698393364, + "mcc_abs": 0.39932125258540485, + "act_iou": 0.08560923101293827, + "act_iou_n_pairs": 12288, + "input_r2": 0.9874251412489551, + "matched_mcc_signed": { + "0.6": 0.5392239242137264, + "0.7": 0.5519042398909687, + "0.8": 0.5624045071468513, + "0.9": 0.5859316319095925 + }, + "matched_mcc_abs": { + "0.6": 0.5386206830869696, + "0.7": 0.5505289132486874, + "0.8": 0.5625832851805398, + "0.9": 0.5864040383189776 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.4147637869907525, + "mcc_abs": 0.4147746334071756, + "act_iou": 0.08912210212555736, + "act_iou_n_pairs": 12288, + "input_r2": 0.98704952291108, + "matched_mcc_signed": { + "0.6": 0.5444276284543527, + "0.7": 0.5542980892050384, + "0.8": 0.558957090463564, + "0.9": 0.5775162106575384 + }, + "matched_mcc_abs": { + "0.6": 0.5427237393358502, + "0.7": 0.553377808580074, + "0.8": 0.5597063188709298, + "0.9": 0.5781317055848446 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.4014848805923182, + "mcc_abs": 0.40150373705422404, + "act_iou": 0.08582858588805446, + "act_iou_n_pairs": 12288, + "input_r2": 0.9870948109538141, + "matched_mcc_signed": { + "0.6": 0.540369712714874, + "0.7": 0.5529784208411271, + "0.8": 0.5615481628054687, + "0.9": 0.5817288928332561 + }, + "matched_mcc_abs": { + "0.6": 0.5395708479129105, + "0.7": 0.5518263326044015, + "0.8": 0.5627321557046862, + "0.9": 0.5824449477433663 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 352.7183355555556, + "l0_std": 9.835488651908037, + "recon_r2_mean": 0.9833455993320648, + "recon_r2_std": 0.0005449917742977583, + "l1_per_token_mean": 155.54782999577665, + "l1_per_token_std": 1.6835683568597046, + "input_r2_mean": 0.9871898250379497, + "input_r2_std": 0.00020504399790072974, + "mcc_signed_mean": 0.4051850715223348, + "mcc_signed_std": 0.008366607845837714, + "mcc_abs_mean": 0.40519987434893484, + "mcc_abs_std": 0.008363481205914302, + "matched_mcc_signed_per_threshold": { + "0.6": 0.5413404217943176, + "0.7": 0.5530602499790448, + "0.8": 0.560969920138628, + "0.9": 0.5817255784667957 + }, + "matched_mcc_signed_band": 0.5592740425946966, + "matched_mcc_abs_per_threshold": { + "0.6": 0.5403050901119101, + "0.7": 0.5519110181443877, + "0.8": 0.5616739199187185, + "0.9": 0.5823268972157295 + }, + "matched_mcc_abs_band": 0.5590542313476864, + "act_iou_mean": 0.08685330634218337, + "act_iou_std": 0.0019678935109149172, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/dict_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x16/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..797dd78f0b9f62a29870a50b7ba1a04e945d2022 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8655, + "0.6": 6939, + "0.7": 5343, + "0.8": 3966, + "0.9": 3042 + }, + "n_dup_clusters_signed": { + "0.5": 2141, + "0.6": 1621, + "0.7": 1065, + "0.8": 562, + "0.9": 270 + }, + "largest_dup_signed": { + "0.5": 914, + "0.6": 832, + "0.7": 832, + "0.8": 678, + "0.9": 233 + }, + "babel_mu1_signed": 0.6928533911705017, + "n_dup_abs": { + "0.5": 8682, + "0.6": 6984, + "0.7": 5382, + "0.8": 3980, + "0.9": 3061 + }, + "n_dup_clusters_abs": { + "0.5": 2139, + "0.6": 1626, + "0.7": 1072, + "0.8": 551, + "0.9": 275 + }, + "largest_dup_abs": { + "0.5": 827, + "0.6": 803, + "0.7": 769, + "0.8": 684, + "0.9": 233 + }, + "babel_mu1_abs": 0.6942043304443359 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8634, + "0.6": 6974, + "0.7": 5382, + "0.8": 3966, + "0.9": 2980 + }, + "n_dup_clusters_signed": { + "0.5": 2105, + "0.6": 1623, + "0.7": 1095, + "0.8": 585, + "0.9": 259 + }, + "largest_dup_signed": { + "0.5": 889, + "0.6": 889, + "0.7": 673, + "0.8": 508, + "0.9": 299 + }, + "babel_mu1_signed": 0.693641722202301, + "n_dup_abs": { + "0.5": 8670, + "0.6": 7031, + "0.7": 5423, + "0.8": 3995, + "0.9": 3003 + }, + "n_dup_clusters_abs": { + "0.5": 2104, + "0.6": 1629, + "0.7": 1095, + "0.8": 576, + "0.9": 260 + }, + "largest_dup_abs": { + "0.5": 1053, + "0.6": 805, + "0.7": 739, + "0.8": 529, + "0.9": 299 + }, + "babel_mu1_abs": 0.6951475143432617 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8649, + "0.6": 6884, + "0.7": 5257, + "0.8": 3951, + "0.9": 3023 + }, + "n_dup_clusters_signed": { + "0.5": 2181, + "0.6": 1619, + "0.7": 1043, + "0.8": 581, + "0.9": 292 + }, + "largest_dup_signed": { + "0.5": 701, + "0.6": 618, + "0.7": 501, + "0.8": 493, + "0.9": 214 + }, + "babel_mu1_signed": 0.6906366348266602, + "n_dup_abs": { + "0.5": 8673, + "0.6": 6933, + "0.7": 5290, + "0.8": 3965, + "0.9": 3044 + }, + "n_dup_clusters_abs": { + "0.5": 2178, + "0.6": 1629, + "0.7": 1043, + "0.8": 571, + "0.9": 287 + }, + "largest_dup_abs": { + "0.5": 822, + "0.6": 797, + "0.7": 797, + "0.8": 444, + "0.9": 214 + }, + "babel_mu1_abs": 0.6920421719551086 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8460356593132019, + "dcs_abs": 0.849884569644928, + "frac_sign_flipped": 0.0244140625, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8108665347099304, + "0.6": 0.8123555779457092, + "0.7": 0.8107208013534546, + "0.8": 0.8103269338607788, + "0.9": 0.8129523396492004 + }, + "unique_frac_signed": { + "0.5": 0.4692789713541667, + "0.6": 0.5658772786458334, + "0.7": 0.6514892578125, + "0.8": 0.7239176432291666, + "0.9": 0.7764892578125 + }, + "dedup_dcs_signed": { + "0.5": 0.38052261331419385, + "0.6": 0.4596935637406811, + "0.7": 0.5281758931669174, + "0.8": 0.5866099642056117, + "0.9": 0.6312487588511431 + }, + "matched_frac_signed": { + "0.5": 0.410400390625, + "0.6": 0.4790852864583333, + "0.7": 0.5068359375, + "0.8": 0.4720865885416667, + "0.9": 0.3621419270833333 + }, + "matched_cos_abs": { + "0.5": 0.8154089450836182, + "0.6": 0.8164052367210388, + "0.7": 0.8139873147010803, + "0.8": 0.8124246001243591, + "0.9": 0.815356969833374 + }, + "unique_frac_abs": { + "0.5": 0.4665934244791667, + "0.6": 0.5621744791666666, + "0.7": 0.6485188802083334, + "0.8": 0.7213541666666666, + "0.9": 0.7750244140625 + }, + "dedup_dcs_abs": { + "0.5": 0.38046445203751017, + "0.6": 0.45896218874258915, + "0.7": 0.5278861418337328, + "0.8": 0.5860458704022069, + "0.9": 0.6319215577968862 + }, + "matched_frac_abs": { + "0.5": 0.410888671875, + "0.6": 0.478759765625, + "0.7": 0.5084635416666666, + "0.8": 0.4715983072916667, + "0.9": 0.362060546875 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8714127540588379, + "dcs_abs": 0.8734491467475891, + "frac_sign_flipped": 0.015787760416666668, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8283454775810242, + "0.6": 0.8341121673583984, + "0.7": 0.8364167809486389, + "0.8": 0.8382395505905151, + "0.9": 0.8423116207122803 + }, + "unique_frac_signed": { + "0.5": 0.4717610677083333, + "0.6": 0.5693766276041666, + "0.7": 0.6544596354166666, + "0.8": 0.724365234375, + "0.9": 0.7760823567708334 + }, + "dedup_dcs_signed": { + "0.5": 0.3907811469349932, + "0.6": 0.47492397289412713, + "0.7": 0.5474010215160281, + "0.8": 0.6071915885258932, + "0.9": 0.6537031877378467 + }, + "matched_frac_signed": { + "0.5": 0.413330078125, + "0.6": 0.4869791666666667, + "0.7": 0.526123046875, + "0.8": 0.5148111979166666, + "0.9": 0.4423828125 + }, + "matched_cos_abs": { + "0.5": 0.8323161602020264, + "0.6": 0.8372353315353394, + "0.7": 0.8390799164772034, + "0.8": 0.8405229449272156, + "0.9": 0.843975841999054 + }, + "unique_frac_abs": { + "0.5": 0.469482421875, + "0.6": 0.566162109375, + "0.7": 0.6518147786458334, + "0.8": 0.7223714192708334, + "0.9": 0.7744547526041666 + }, + "dedup_dcs_abs": { + "0.5": 0.39075780665734783, + "0.6": 0.4740109213453252, + "0.7": 0.5469246900247526, + "0.8": 0.6071697526567732, + "0.9": 0.6536211019192706 + }, + "matched_frac_abs": { + "0.5": 0.4138997395833333, + "0.6": 0.488037109375, + "0.7": 0.52783203125, + "0.8": 0.51416015625, + "0.9": 0.44091796875 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.848410427570343, + "dcs_abs": 0.8519936203956604, + "frac_sign_flipped": 0.022542317708333332, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8142401576042175, + "0.6": 0.817314088344574, + "0.7": 0.8153243064880371, + "0.8": 0.8121744394302368, + "0.9": 0.8141469359397888 + }, + "unique_frac_signed": { + "0.5": 0.4711507161458333, + "0.6": 0.5680338541666666, + "0.7": 0.6540934244791666, + "0.8": 0.7253011067708334, + "0.9": 0.7781575520833334 + }, + "dedup_dcs_signed": { + "0.5": 0.3836298333699233, + "0.6": 0.46426207166708383, + "0.7": 0.5332982676918618, + "0.8": 0.589071019809732, + "0.9": 0.6335345867070525 + }, + "matched_frac_signed": { + "0.5": 0.41259765625, + "0.6": 0.4830729166666667, + "0.7": 0.510986328125, + "0.8": 0.4785970052083333, + "0.9": 0.37451171875 + }, + "matched_cos_abs": { + "0.5": 0.818837583065033, + "0.6": 0.8209933042526245, + "0.7": 0.818362832069397, + "0.8": 0.8152377009391785, + "0.9": 0.8166646361351013 + }, + "unique_frac_abs": { + "0.5": 0.4685465494791667, + "0.6": 0.5643717447916666, + "0.7": 0.6510823567708334, + "0.8": 0.7227783203125, + "0.9": 0.7762044270833334 + }, + "dedup_dcs_abs": { + "0.5": 0.38366352412898175, + "0.6": 0.4633454235833293, + "0.7": 0.5328216013973968, + "0.8": 0.5892361361402436, + "0.9": 0.6338987060104653 + }, + "matched_frac_abs": { + "0.5": 0.4126790364583333, + "0.6": 0.4825032552083333, + "0.7": 0.51123046875, + "0.8": 0.4768880208333333, + "0.9": 0.3736165364583333 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.6923772493998209, + "babel_mu1_signed_std": 0.0015580984478361045, + "n_dup_signed_per_threshold": { + "0.5": 8646.0, + "0.6": 6932.333333333333, + "0.7": 5327.333333333333, + "0.8": 3961.0, + "0.9": 3015.0 + }, + "n_dup_signed_band": 4808.916666666666, + "n_dup_signed_band_std": 26.16812628625392, + "n_dup_clusters_signed_per_threshold": { + "0.5": 2142.3333333333335, + "0.6": 1621.0, + "0.7": 1067.6666666666667, + "0.8": 576.0, + "0.9": 273.6666666666667 + }, + "n_dup_clusters_signed_band": 884.5833333333334, + "n_dup_clusters_signed_band_std": 5.54714641354754, + "largest_dup_signed_per_threshold": { + "0.5": 834.6666666666666, + "0.6": 779.6666666666666, + "0.7": 668.6666666666666, + "0.8": 559.6666666666666, + "0.9": 248.66666666666666 + }, + "largest_dup_signed_band": 564.1666666666666, + "largest_dup_signed_band_std": 96.73234119638236, + "dcs_signed_mean": 0.8552862803141276, + "dcs_signed_std": 0.014016320760024861, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.38497786453970345, + "0.6": 0.4662932027672973, + "0.7": 0.5362917274582691, + "0.8": 0.5942908575137457, + "0.9": 0.6394955110986807 + }, + "dedup_dcs_signed_band": 0.5590928247094983, + "dedup_dcs_signed_band_std": 0.010302295727614024, + "matched_cos_signed_per_threshold": { + "0.5": 0.8178173899650574, + "0.6": 0.8212606112162272, + "0.7": 0.8208206295967102, + "0.8": 0.8202469746271769, + "0.9": 0.8231369654337565 + }, + "matched_cos_signed_band": 0.8213662952184676, + "matched_cos_signed_band_std": 0.014293149763982807, + "unique_frac_signed_per_threshold": { + "0.5": 0.4707302517361111, + "0.6": 0.5677625868055555, + "0.7": 0.653347439236111, + "0.8": 0.7245279947916666, + "0.9": 0.7769097222222223 + }, + "unique_frac_signed_band": 0.6806369357638888, + "unique_frac_signed_band_std": 0.0010464030773646793, + "matched_frac_signed_per_threshold": { + "0.5": 0.412109375, + "0.6": 0.4830457899305556, + "0.7": 0.5146484375, + "0.8": 0.48849826388888884, + "0.9": 0.39301215277777773 + }, + "matched_frac_signed_band": 0.4698011610243055, + "matched_frac_signed_band_std": 0.020008987227650564, + "babel_mu1_abs_mean": 0.6937980055809021, + "babel_mu1_abs_std": 0.001592046719759963, + "n_dup_abs_per_threshold": { + "0.5": 8675.0, + "0.6": 6982.666666666667, + "0.7": 5365.0, + "0.8": 3980.0, + "0.9": 3036.0 + }, + "n_dup_abs_band": 4840.916666666667, + "n_dup_abs_band_std": 29.056338952685234, + "n_dup_clusters_abs_per_threshold": { + "0.5": 2140.3333333333335, + "0.6": 1628.0, + "0.7": 1070.0, + "0.8": 566.0, + "0.9": 274.0 + }, + "n_dup_clusters_abs_band": 884.5, + "n_dup_clusters_abs_band_std": 4.8218253804964775, + "largest_dup_abs_per_threshold": { + "0.5": 900.6666666666666, + "0.6": 801.6666666666666, + "0.7": 768.3333333333334, + "0.8": 552.3333333333334, + "0.9": 248.66666666666666 + }, + "largest_dup_abs_band": 592.75, + "largest_dup_abs_band_std": 29.625791128677054, + "dcs_abs_mean": 0.8584424455960592, + "dcs_abs_std": 0.013038896937846736, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.38496192760794656, + "0.6": 0.4654395112237479, + "0.7": 0.5358774777519607, + "0.8": 0.5941505863997413, + "0.9": 0.6398137885755407 + }, + "dedup_dcs_abs_band": 0.5588203409877476, + "dedup_dcs_abs_band_std": 0.010217394660361043, + "matched_cos_abs_per_threshold": { + "0.5": 0.8221875627835592, + "0.6": 0.8248779575030009, + "0.7": 0.8238100210825602, + "0.8": 0.8227284153302511, + "0.9": 0.8253324826558431 + }, + "matched_cos_abs_band": 0.8241872191429138, + "matched_cos_abs_band_std": 0.013966608514550128, + "unique_frac_abs_per_threshold": { + "0.5": 0.46820746527777785, + "0.6": 0.564236111111111, + "0.7": 0.6504720052083334, + "0.8": 0.72216796875, + "0.9": 0.7752278645833334 + }, + "unique_frac_abs_band": 0.6780259874131944, + "unique_frac_abs_band_std": 0.0010904233032884723, + "matched_frac_abs_per_threshold": { + "0.5": 0.4124891493055555, + "0.6": 0.48310004340277773, + "0.7": 0.5158420138888888, + "0.8": 0.487548828125, + "0.9": 0.3921983506944444 + }, + "matched_frac_abs_band": 0.46967230902777773, + "matched_frac_abs_band_std": 0.02018668355975957, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/config.json b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..582c8dce741510337aae6bc33e8db9b021f25bc1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/lock.lock b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/model.safetensors b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..84541d8b4db74a7094607c2fa226cd119b681ef0 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca64feaf4a78eaac4082e45ec5f454e6a77b21c6cd3008d1ec9ef27dfa72ebe6 +size 75550008 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfb500a1e757ec803c90b616521553d5fb7a30a3 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63e55c4d2231e9f350ce2c654a615ff0fbfc44c3795a0bc082d7b0990161aa45 +size 99648 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7887da6d40a9f6a65e70e8a75e15ef67d99c8538 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3042e8b83d6f5ce39d9d63a8787ec95fba7b4801db6b426a86d47f09c993f3e7 +size 50471 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..987a3072d8adcf454d92de15604775947df65e9d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.03828061523438, + "l0_per_token_0.001": 32.03828061523438, + "l0_per_token_0.01": 32.03828061523438, + "l0_per_token_0.05": 32.03828061523438, + "l0_per_token_0.1": 32.03828061523438, + "l0_per_token_0.5": 31.459601025390626, + "l0_per_token_1.0": 15.88932890625, + "l1_per_token": 60.984214639496244, + "r2_score": 0.9197678115707759, + "l2_loss": 0.053416263354113445 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5151223a0a1f4237bca972f3ff4eefc033ca032a --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1c25daea99134583d3add9462a686544c76e3636d325f71725488870d274899 +size 50382 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfaf755af6248b5b802b85b9077dc1a85b09854c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2850e640a83d64efadc85dff2bccfa91642dfedbe253d24ab3b5fed9a5db34f +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ac2495e34c1689a790642aef613ac159fa2a1de --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da06c18f905e66ed53e9e7cae0de72ca5acc6501962f3c79c809b927341440e0 +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/config.json b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8465153a9e9a1f2031f87935b38abdc81b8f3232 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/lock.lock b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/model.safetensors b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a8d089cce87f1716e70fc858e70c44982f8e7186 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36b2b7bc941d2e08f3261ad9970fe72d4b2fda7326b285b9ecb8caaa20741234 +size 75550008 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..833e753ac7c30b9a1ab87b82f7f33e56877dd74d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a86a6850eacd5cf11024050560981076ce4e86dea2c9e88e37048ff98af4ff6 +size 99648 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..1651b125fc57b029b94b5d1eab8cfcdd27a62269 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3dac1be181bedf6465b34360e34bab47f270941332ae58ca2c9ee8bc789bcb1 +size 50471 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..8107d845e0d99e0dbac5aa99e41ff9af5013561c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.621306640625, + "l0_per_token_0.001": 31.621306640625, + "l0_per_token_0.01": 31.621306640625, + "l0_per_token_0.05": 31.621306640625, + "l0_per_token_0.1": 31.621306640625, + "l0_per_token_0.5": 31.088189208984375, + "l0_per_token_1.0": 15.836713134765626, + "l1_per_token": 60.57542369279035, + "r2_score": 0.9198098326599718, + "l2_loss": 0.05338828693204572 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..aca19f76a133632c86cbdd2cf0b173270a918e78 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8f265eb2ebaa7f69592fe7ccf1b913bbd4e0c9c9b64ca28bc1e6720edbae952 +size 50382 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1db008aead1355c41a831a13c432e98cf198cdd --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:019c4a5720c15455f35641c079925d8136e1d5bac61785ca7c5da29d32fff851 +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5290e110750731bbcde2658c4a8d1aba2a7335b --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d9139e352cdf2cb3d40a02d4cd8f2af13ec356b63580c63e754913b3057433b +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/config.json b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..859d3bae315c9ef5dc1a4ece37711d5e66c114f1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/lock.lock b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/model.safetensors b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..00d2048bcffd6adb83cf3e50c2379e2ab52fef82 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:062c936841ac8e0b34721eec7f7cc9a9faa05dbd11fae1007a18f9581c94c14d +size 75550008 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..096b298cfc1101a0819b6446517045a069aa23f0 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bffb088ed9312df38e642f926da185f2b79cbe04455e4b84dda59ac6d9b33f0d +size 99648 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2b7e6aa0e8d9d16604813d323708c250215176d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4a961aa9c3193c93db12860e4f44dd13d8897e5df32d2f43b3752d1f5360607 +size 50471 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e175dff7ff72e607cfdfe8ee787e9ed0c23d93c2 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 30.96504931640625, + "l0_per_token_0.001": 30.96504931640625, + "l0_per_token_0.01": 30.96504931640625, + "l0_per_token_0.05": 30.96504931640625, + "l0_per_token_0.1": 30.96504931640625, + "l0_per_token_0.5": 30.4920673828125, + "l0_per_token_1.0": 15.85408134765625, + "l1_per_token": 60.35047452306022, + "r2_score": 0.9189396134275296, + "l2_loss": 0.05396765365014229 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..28b278b220ba1cbf39f3bc18dec1e933911544bc --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b16d9bf8db2727909803ab3f997fa529513a1418084f126603313133b63a7c33 +size 50382 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..09c87dd8c3487e5807cb2c755845a011e6e99657 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c8a03084d5b49df589da4d776b121abd6a13a5abcd0d5a1748be6a38862ba73 +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dced1c9e5c906272c657860bcc8c235979171f7e --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c443d9a1c2a39b93121def1f1f210068194a97fa3e7fe94c8d0b59b95aaeed8 +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/code_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x16/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..7c0019777f9951bcdedc141673456017af0c1340 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 355.9318166666667, + "recon_r2": 0.9834826408282966, + "l1_per_token": 156.23972150447466 + }, + "seed1": { + "l0": 360.54515166666664, + "recon_r2": 0.9838089909407193, + "l1_per_token": 156.77521415513394 + }, + "seed2": { + "l0": 341.67803833333335, + "recon_r2": 0.9827451662271783, + "l1_per_token": 153.6285543277214 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.39930654698393364, + "mcc_abs": 0.39932125258540485, + "act_iou": 0.08560923101293827, + "act_iou_n_pairs": 12288, + "input_r2": 0.9874251412489551, + "matched_mcc_signed": { + "0.6": 0.5392239242137264, + "0.7": 0.5519042398909687, + "0.8": 0.5624045071468513, + "0.9": 0.5859316319095925 + }, + "matched_mcc_abs": { + "0.6": 0.5386206830869696, + "0.7": 0.5505289132486874, + "0.8": 0.5625832851805398, + "0.9": 0.5864040383189776 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.4147637869907525, + "mcc_abs": 0.4147746334071756, + "act_iou": 0.08912210212555736, + "act_iou_n_pairs": 12288, + "input_r2": 0.98704952291108, + "matched_mcc_signed": { + "0.6": 0.5444276284543527, + "0.7": 0.5542980892050384, + "0.8": 0.558957090463564, + "0.9": 0.5775162106575384 + }, + "matched_mcc_abs": { + "0.6": 0.5427237393358502, + "0.7": 0.553377808580074, + "0.8": 0.5597063188709298, + "0.9": 0.5781317055848446 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.4014848805923182, + "mcc_abs": 0.40150373705422404, + "act_iou": 0.08582858588805446, + "act_iou_n_pairs": 12288, + "input_r2": 0.9870948109538141, + "matched_mcc_signed": { + "0.6": 0.540369712714874, + "0.7": 0.5529784208411271, + "0.8": 0.5615481628054687, + "0.9": 0.5817288928332561 + }, + "matched_mcc_abs": { + "0.6": 0.5395708479129105, + "0.7": 0.5518263326044015, + "0.8": 0.5627321557046862, + "0.9": 0.5824449477433663 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 352.7183355555556, + "l0_std": 9.835488651908037, + "recon_r2_mean": 0.9833455993320648, + "recon_r2_std": 0.0005449917742977583, + "l1_per_token_mean": 155.54782999577665, + "l1_per_token_std": 1.6835683568597046, + "input_r2_mean": 0.9871898250379497, + "input_r2_std": 0.00020504399790072974, + "mcc_signed_mean": 0.4051850715223348, + "mcc_signed_std": 0.008366607845837714, + "mcc_abs_mean": 0.40519987434893484, + "mcc_abs_std": 0.008363481205914302, + "matched_mcc_signed_per_threshold": { + "0.6": 0.5413404217943176, + "0.7": 0.5530602499790448, + "0.8": 0.560969920138628, + "0.9": 0.5817255784667957 + }, + "matched_mcc_signed_band": 0.5592740425946966, + "matched_mcc_abs_per_threshold": { + "0.6": 0.5403050901119101, + "0.7": 0.5519110181443877, + "0.8": 0.5616739199187185, + "0.9": 0.5823268972157295 + }, + "matched_mcc_abs_band": 0.5590542313476864, + "act_iou_mean": 0.08685330634218337, + "act_iou_std": 0.0019678935109149172, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/dict_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x16/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..797dd78f0b9f62a29870a50b7ba1a04e945d2022 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8655, + "0.6": 6939, + "0.7": 5343, + "0.8": 3966, + "0.9": 3042 + }, + "n_dup_clusters_signed": { + "0.5": 2141, + "0.6": 1621, + "0.7": 1065, + "0.8": 562, + "0.9": 270 + }, + "largest_dup_signed": { + "0.5": 914, + "0.6": 832, + "0.7": 832, + "0.8": 678, + "0.9": 233 + }, + "babel_mu1_signed": 0.6928533911705017, + "n_dup_abs": { + "0.5": 8682, + "0.6": 6984, + "0.7": 5382, + "0.8": 3980, + "0.9": 3061 + }, + "n_dup_clusters_abs": { + "0.5": 2139, + "0.6": 1626, + "0.7": 1072, + "0.8": 551, + "0.9": 275 + }, + "largest_dup_abs": { + "0.5": 827, + "0.6": 803, + "0.7": 769, + "0.8": 684, + "0.9": 233 + }, + "babel_mu1_abs": 0.6942043304443359 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8634, + "0.6": 6974, + "0.7": 5382, + "0.8": 3966, + "0.9": 2980 + }, + "n_dup_clusters_signed": { + "0.5": 2105, + "0.6": 1623, + "0.7": 1095, + "0.8": 585, + "0.9": 259 + }, + "largest_dup_signed": { + "0.5": 889, + "0.6": 889, + "0.7": 673, + "0.8": 508, + "0.9": 299 + }, + "babel_mu1_signed": 0.693641722202301, + "n_dup_abs": { + "0.5": 8670, + "0.6": 7031, + "0.7": 5423, + "0.8": 3995, + "0.9": 3003 + }, + "n_dup_clusters_abs": { + "0.5": 2104, + "0.6": 1629, + "0.7": 1095, + "0.8": 576, + "0.9": 260 + }, + "largest_dup_abs": { + "0.5": 1053, + "0.6": 805, + "0.7": 739, + "0.8": 529, + "0.9": 299 + }, + "babel_mu1_abs": 0.6951475143432617 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8649, + "0.6": 6884, + "0.7": 5257, + "0.8": 3951, + "0.9": 3023 + }, + "n_dup_clusters_signed": { + "0.5": 2181, + "0.6": 1619, + "0.7": 1043, + "0.8": 581, + "0.9": 292 + }, + "largest_dup_signed": { + "0.5": 701, + "0.6": 618, + "0.7": 501, + "0.8": 493, + "0.9": 214 + }, + "babel_mu1_signed": 0.6906366348266602, + "n_dup_abs": { + "0.5": 8673, + "0.6": 6933, + "0.7": 5290, + "0.8": 3965, + "0.9": 3044 + }, + "n_dup_clusters_abs": { + "0.5": 2178, + "0.6": 1629, + "0.7": 1043, + "0.8": 571, + "0.9": 287 + }, + "largest_dup_abs": { + "0.5": 822, + "0.6": 797, + "0.7": 797, + "0.8": 444, + "0.9": 214 + }, + "babel_mu1_abs": 0.6920421719551086 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8460356593132019, + "dcs_abs": 0.849884569644928, + "frac_sign_flipped": 0.0244140625, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8108665347099304, + "0.6": 0.8123555779457092, + "0.7": 0.8107208013534546, + "0.8": 0.8103269338607788, + "0.9": 0.8129523396492004 + }, + "unique_frac_signed": { + "0.5": 0.4692789713541667, + "0.6": 0.5658772786458334, + "0.7": 0.6514892578125, + "0.8": 0.7239176432291666, + "0.9": 0.7764892578125 + }, + "dedup_dcs_signed": { + "0.5": 0.38052261331419385, + "0.6": 0.4596935637406811, + "0.7": 0.5281758931669174, + "0.8": 0.5866099642056117, + "0.9": 0.6312487588511431 + }, + "matched_frac_signed": { + "0.5": 0.410400390625, + "0.6": 0.4790852864583333, + "0.7": 0.5068359375, + "0.8": 0.4720865885416667, + "0.9": 0.3621419270833333 + }, + "matched_cos_abs": { + "0.5": 0.8154089450836182, + "0.6": 0.8164052367210388, + "0.7": 0.8139873147010803, + "0.8": 0.8124246001243591, + "0.9": 0.815356969833374 + }, + "unique_frac_abs": { + "0.5": 0.4665934244791667, + "0.6": 0.5621744791666666, + "0.7": 0.6485188802083334, + "0.8": 0.7213541666666666, + "0.9": 0.7750244140625 + }, + "dedup_dcs_abs": { + "0.5": 0.38046445203751017, + "0.6": 0.45896218874258915, + "0.7": 0.5278861418337328, + "0.8": 0.5860458704022069, + "0.9": 0.6319215577968862 + }, + "matched_frac_abs": { + "0.5": 0.410888671875, + "0.6": 0.478759765625, + "0.7": 0.5084635416666666, + "0.8": 0.4715983072916667, + "0.9": 0.362060546875 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8714127540588379, + "dcs_abs": 0.8734491467475891, + "frac_sign_flipped": 0.015787760416666668, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8283454775810242, + "0.6": 0.8341121673583984, + "0.7": 0.8364167809486389, + "0.8": 0.8382395505905151, + "0.9": 0.8423116207122803 + }, + "unique_frac_signed": { + "0.5": 0.4717610677083333, + "0.6": 0.5693766276041666, + "0.7": 0.6544596354166666, + "0.8": 0.724365234375, + "0.9": 0.7760823567708334 + }, + "dedup_dcs_signed": { + "0.5": 0.3907811469349932, + "0.6": 0.47492397289412713, + "0.7": 0.5474010215160281, + "0.8": 0.6071915885258932, + "0.9": 0.6537031877378467 + }, + "matched_frac_signed": { + "0.5": 0.413330078125, + "0.6": 0.4869791666666667, + "0.7": 0.526123046875, + "0.8": 0.5148111979166666, + "0.9": 0.4423828125 + }, + "matched_cos_abs": { + "0.5": 0.8323161602020264, + "0.6": 0.8372353315353394, + "0.7": 0.8390799164772034, + "0.8": 0.8405229449272156, + "0.9": 0.843975841999054 + }, + "unique_frac_abs": { + "0.5": 0.469482421875, + "0.6": 0.566162109375, + "0.7": 0.6518147786458334, + "0.8": 0.7223714192708334, + "0.9": 0.7744547526041666 + }, + "dedup_dcs_abs": { + "0.5": 0.39075780665734783, + "0.6": 0.4740109213453252, + "0.7": 0.5469246900247526, + "0.8": 0.6071697526567732, + "0.9": 0.6536211019192706 + }, + "matched_frac_abs": { + "0.5": 0.4138997395833333, + "0.6": 0.488037109375, + "0.7": 0.52783203125, + "0.8": 0.51416015625, + "0.9": 0.44091796875 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.848410427570343, + "dcs_abs": 0.8519936203956604, + "frac_sign_flipped": 0.022542317708333332, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8142401576042175, + "0.6": 0.817314088344574, + "0.7": 0.8153243064880371, + "0.8": 0.8121744394302368, + "0.9": 0.8141469359397888 + }, + "unique_frac_signed": { + "0.5": 0.4711507161458333, + "0.6": 0.5680338541666666, + "0.7": 0.6540934244791666, + "0.8": 0.7253011067708334, + "0.9": 0.7781575520833334 + }, + "dedup_dcs_signed": { + "0.5": 0.3836298333699233, + "0.6": 0.46426207166708383, + "0.7": 0.5332982676918618, + "0.8": 0.589071019809732, + "0.9": 0.6335345867070525 + }, + "matched_frac_signed": { + "0.5": 0.41259765625, + "0.6": 0.4830729166666667, + "0.7": 0.510986328125, + "0.8": 0.4785970052083333, + "0.9": 0.37451171875 + }, + "matched_cos_abs": { + "0.5": 0.818837583065033, + "0.6": 0.8209933042526245, + "0.7": 0.818362832069397, + "0.8": 0.8152377009391785, + "0.9": 0.8166646361351013 + }, + "unique_frac_abs": { + "0.5": 0.4685465494791667, + "0.6": 0.5643717447916666, + "0.7": 0.6510823567708334, + "0.8": 0.7227783203125, + "0.9": 0.7762044270833334 + }, + "dedup_dcs_abs": { + "0.5": 0.38366352412898175, + "0.6": 0.4633454235833293, + "0.7": 0.5328216013973968, + "0.8": 0.5892361361402436, + "0.9": 0.6338987060104653 + }, + "matched_frac_abs": { + "0.5": 0.4126790364583333, + "0.6": 0.4825032552083333, + "0.7": 0.51123046875, + "0.8": 0.4768880208333333, + "0.9": 0.3736165364583333 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.6923772493998209, + "babel_mu1_signed_std": 0.0015580984478361045, + "n_dup_signed_per_threshold": { + "0.5": 8646.0, + "0.6": 6932.333333333333, + "0.7": 5327.333333333333, + "0.8": 3961.0, + "0.9": 3015.0 + }, + "n_dup_signed_band": 4808.916666666666, + "n_dup_signed_band_std": 26.16812628625392, + "n_dup_clusters_signed_per_threshold": { + "0.5": 2142.3333333333335, + "0.6": 1621.0, + "0.7": 1067.6666666666667, + "0.8": 576.0, + "0.9": 273.6666666666667 + }, + "n_dup_clusters_signed_band": 884.5833333333334, + "n_dup_clusters_signed_band_std": 5.54714641354754, + "largest_dup_signed_per_threshold": { + "0.5": 834.6666666666666, + "0.6": 779.6666666666666, + "0.7": 668.6666666666666, + "0.8": 559.6666666666666, + "0.9": 248.66666666666666 + }, + "largest_dup_signed_band": 564.1666666666666, + "largest_dup_signed_band_std": 96.73234119638236, + "dcs_signed_mean": 0.8552862803141276, + "dcs_signed_std": 0.014016320760024861, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.38497786453970345, + "0.6": 0.4662932027672973, + "0.7": 0.5362917274582691, + "0.8": 0.5942908575137457, + "0.9": 0.6394955110986807 + }, + "dedup_dcs_signed_band": 0.5590928247094983, + "dedup_dcs_signed_band_std": 0.010302295727614024, + "matched_cos_signed_per_threshold": { + "0.5": 0.8178173899650574, + "0.6": 0.8212606112162272, + "0.7": 0.8208206295967102, + "0.8": 0.8202469746271769, + "0.9": 0.8231369654337565 + }, + "matched_cos_signed_band": 0.8213662952184676, + "matched_cos_signed_band_std": 0.014293149763982807, + "unique_frac_signed_per_threshold": { + "0.5": 0.4707302517361111, + "0.6": 0.5677625868055555, + "0.7": 0.653347439236111, + "0.8": 0.7245279947916666, + "0.9": 0.7769097222222223 + }, + "unique_frac_signed_band": 0.6806369357638888, + "unique_frac_signed_band_std": 0.0010464030773646793, + "matched_frac_signed_per_threshold": { + "0.5": 0.412109375, + "0.6": 0.4830457899305556, + "0.7": 0.5146484375, + "0.8": 0.48849826388888884, + "0.9": 0.39301215277777773 + }, + "matched_frac_signed_band": 0.4698011610243055, + "matched_frac_signed_band_std": 0.020008987227650564, + "babel_mu1_abs_mean": 0.6937980055809021, + "babel_mu1_abs_std": 0.001592046719759963, + "n_dup_abs_per_threshold": { + "0.5": 8675.0, + "0.6": 6982.666666666667, + "0.7": 5365.0, + "0.8": 3980.0, + "0.9": 3036.0 + }, + "n_dup_abs_band": 4840.916666666667, + "n_dup_abs_band_std": 29.056338952685234, + "n_dup_clusters_abs_per_threshold": { + "0.5": 2140.3333333333335, + "0.6": 1628.0, + "0.7": 1070.0, + "0.8": 566.0, + "0.9": 274.0 + }, + "n_dup_clusters_abs_band": 884.5, + "n_dup_clusters_abs_band_std": 4.8218253804964775, + "largest_dup_abs_per_threshold": { + "0.5": 900.6666666666666, + "0.6": 801.6666666666666, + "0.7": 768.3333333333334, + "0.8": 552.3333333333334, + "0.9": 248.66666666666666 + }, + "largest_dup_abs_band": 592.75, + "largest_dup_abs_band_std": 29.625791128677054, + "dcs_abs_mean": 0.8584424455960592, + "dcs_abs_std": 0.013038896937846736, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.38496192760794656, + "0.6": 0.4654395112237479, + "0.7": 0.5358774777519607, + "0.8": 0.5941505863997413, + "0.9": 0.6398137885755407 + }, + "dedup_dcs_abs_band": 0.5588203409877476, + "dedup_dcs_abs_band_std": 0.010217394660361043, + "matched_cos_abs_per_threshold": { + "0.5": 0.8221875627835592, + "0.6": 0.8248779575030009, + "0.7": 0.8238100210825602, + "0.8": 0.8227284153302511, + "0.9": 0.8253324826558431 + }, + "matched_cos_abs_band": 0.8241872191429138, + "matched_cos_abs_band_std": 0.013966608514550128, + "unique_frac_abs_per_threshold": { + "0.5": 0.46820746527777785, + "0.6": 0.564236111111111, + "0.7": 0.6504720052083334, + "0.8": 0.72216796875, + "0.9": 0.7752278645833334 + }, + "unique_frac_abs_band": 0.6780259874131944, + "unique_frac_abs_band_std": 0.0010904233032884723, + "matched_frac_abs_per_threshold": { + "0.5": 0.4124891493055555, + "0.6": 0.48310004340277773, + "0.7": 0.5158420138888888, + "0.8": 0.487548828125, + "0.9": 0.3921983506944444 + }, + "matched_frac_abs_band": 0.46967230902777773, + "matched_frac_abs_band_std": 0.02018668355975957, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/config.json b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..582c8dce741510337aae6bc33e8db9b021f25bc1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/lock.lock b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/model.safetensors b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..84541d8b4db74a7094607c2fa226cd119b681ef0 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca64feaf4a78eaac4082e45ec5f454e6a77b21c6cd3008d1ec9ef27dfa72ebe6 +size 75550008 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfb500a1e757ec803c90b616521553d5fb7a30a3 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63e55c4d2231e9f350ce2c654a615ff0fbfc44c3795a0bc082d7b0990161aa45 +size 99648 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7887da6d40a9f6a65e70e8a75e15ef67d99c8538 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3042e8b83d6f5ce39d9d63a8787ec95fba7b4801db6b426a86d47f09c993f3e7 +size 50471 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..987a3072d8adcf454d92de15604775947df65e9d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.03828061523438, + "l0_per_token_0.001": 32.03828061523438, + "l0_per_token_0.01": 32.03828061523438, + "l0_per_token_0.05": 32.03828061523438, + "l0_per_token_0.1": 32.03828061523438, + "l0_per_token_0.5": 31.459601025390626, + "l0_per_token_1.0": 15.88932890625, + "l1_per_token": 60.984214639496244, + "r2_score": 0.9197678115707759, + "l2_loss": 0.053416263354113445 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5151223a0a1f4237bca972f3ff4eefc033ca032a --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1c25daea99134583d3add9462a686544c76e3636d325f71725488870d274899 +size 50382 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfaf755af6248b5b802b85b9077dc1a85b09854c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2850e640a83d64efadc85dff2bccfa91642dfedbe253d24ab3b5fed9a5db34f +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ac2495e34c1689a790642aef613ac159fa2a1de --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da06c18f905e66ed53e9e7cae0de72ca5acc6501962f3c79c809b927341440e0 +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/config.json b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8465153a9e9a1f2031f87935b38abdc81b8f3232 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/lock.lock b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/model.safetensors b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a8d089cce87f1716e70fc858e70c44982f8e7186 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36b2b7bc941d2e08f3261ad9970fe72d4b2fda7326b285b9ecb8caaa20741234 +size 75550008 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..833e753ac7c30b9a1ab87b82f7f33e56877dd74d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a86a6850eacd5cf11024050560981076ce4e86dea2c9e88e37048ff98af4ff6 +size 99648 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..1651b125fc57b029b94b5d1eab8cfcdd27a62269 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3dac1be181bedf6465b34360e34bab47f270941332ae58ca2c9ee8bc789bcb1 +size 50471 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..8107d845e0d99e0dbac5aa99e41ff9af5013561c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.621306640625, + "l0_per_token_0.001": 31.621306640625, + "l0_per_token_0.01": 31.621306640625, + "l0_per_token_0.05": 31.621306640625, + "l0_per_token_0.1": 31.621306640625, + "l0_per_token_0.5": 31.088189208984375, + "l0_per_token_1.0": 15.836713134765626, + "l1_per_token": 60.57542369279035, + "r2_score": 0.9198098326599718, + "l2_loss": 0.05338828693204572 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..aca19f76a133632c86cbdd2cf0b173270a918e78 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8f265eb2ebaa7f69592fe7ccf1b913bbd4e0c9c9b64ca28bc1e6720edbae952 +size 50382 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1db008aead1355c41a831a13c432e98cf198cdd --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:019c4a5720c15455f35641c079925d8136e1d5bac61785ca7c5da29d32fff851 +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5290e110750731bbcde2658c4a8d1aba2a7335b --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d9139e352cdf2cb3d40a02d4cd8f2af13ec356b63580c63e754913b3057433b +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/config.json b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..859d3bae315c9ef5dc1a4ece37711d5e66c114f1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/lock.lock b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/model.safetensors b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..00d2048bcffd6adb83cf3e50c2379e2ab52fef82 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:062c936841ac8e0b34721eec7f7cc9a9faa05dbd11fae1007a18f9581c94c14d +size 75550008 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..096b298cfc1101a0819b6446517045a069aa23f0 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bffb088ed9312df38e642f926da185f2b79cbe04455e4b84dda59ac6d9b33f0d +size 99648 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2b7e6aa0e8d9d16604813d323708c250215176d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4a961aa9c3193c93db12860e4f44dd13d8897e5df32d2f43b3752d1f5360607 +size 50471 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e175dff7ff72e607cfdfe8ee787e9ed0c23d93c2 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 30.96504931640625, + "l0_per_token_0.001": 30.96504931640625, + "l0_per_token_0.01": 30.96504931640625, + "l0_per_token_0.05": 30.96504931640625, + "l0_per_token_0.1": 30.96504931640625, + "l0_per_token_0.5": 30.4920673828125, + "l0_per_token_1.0": 15.85408134765625, + "l1_per_token": 60.35047452306022, + "r2_score": 0.9189396134275296, + "l2_loss": 0.05396765365014229 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..28b278b220ba1cbf39f3bc18dec1e933911544bc --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b16d9bf8db2727909803ab3f997fa529513a1418084f126603313133b63a7c33 +size 50382 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..09c87dd8c3487e5807cb2c755845a011e6e99657 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c8a03084d5b49df589da4d776b121abd6a13a5abcd0d5a1748be6a38862ba73 +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dced1c9e5c906272c657860bcc8c235979171f7e --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c443d9a1c2a39b93121def1f1f210068194a97fa3e7fe94c8d0b59b95aaeed8 +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/code_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x16/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..7c0019777f9951bcdedc141673456017af0c1340 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 355.9318166666667, + "recon_r2": 0.9834826408282966, + "l1_per_token": 156.23972150447466 + }, + "seed1": { + "l0": 360.54515166666664, + "recon_r2": 0.9838089909407193, + "l1_per_token": 156.77521415513394 + }, + "seed2": { + "l0": 341.67803833333335, + "recon_r2": 0.9827451662271783, + "l1_per_token": 153.6285543277214 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.39930654698393364, + "mcc_abs": 0.39932125258540485, + "act_iou": 0.08560923101293827, + "act_iou_n_pairs": 12288, + "input_r2": 0.9874251412489551, + "matched_mcc_signed": { + "0.6": 0.5392239242137264, + "0.7": 0.5519042398909687, + "0.8": 0.5624045071468513, + "0.9": 0.5859316319095925 + }, + "matched_mcc_abs": { + "0.6": 0.5386206830869696, + "0.7": 0.5505289132486874, + "0.8": 0.5625832851805398, + "0.9": 0.5864040383189776 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.4147637869907525, + "mcc_abs": 0.4147746334071756, + "act_iou": 0.08912210212555736, + "act_iou_n_pairs": 12288, + "input_r2": 0.98704952291108, + "matched_mcc_signed": { + "0.6": 0.5444276284543527, + "0.7": 0.5542980892050384, + "0.8": 0.558957090463564, + "0.9": 0.5775162106575384 + }, + "matched_mcc_abs": { + "0.6": 0.5427237393358502, + "0.7": 0.553377808580074, + "0.8": 0.5597063188709298, + "0.9": 0.5781317055848446 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.4014848805923182, + "mcc_abs": 0.40150373705422404, + "act_iou": 0.08582858588805446, + "act_iou_n_pairs": 12288, + "input_r2": 0.9870948109538141, + "matched_mcc_signed": { + "0.6": 0.540369712714874, + "0.7": 0.5529784208411271, + "0.8": 0.5615481628054687, + "0.9": 0.5817288928332561 + }, + "matched_mcc_abs": { + "0.6": 0.5395708479129105, + "0.7": 0.5518263326044015, + "0.8": 0.5627321557046862, + "0.9": 0.5824449477433663 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 352.7183355555556, + "l0_std": 9.835488651908037, + "recon_r2_mean": 0.9833455993320648, + "recon_r2_std": 0.0005449917742977583, + "l1_per_token_mean": 155.54782999577665, + "l1_per_token_std": 1.6835683568597046, + "input_r2_mean": 0.9871898250379497, + "input_r2_std": 0.00020504399790072974, + "mcc_signed_mean": 0.4051850715223348, + "mcc_signed_std": 0.008366607845837714, + "mcc_abs_mean": 0.40519987434893484, + "mcc_abs_std": 0.008363481205914302, + "matched_mcc_signed_per_threshold": { + "0.6": 0.5413404217943176, + "0.7": 0.5530602499790448, + "0.8": 0.560969920138628, + "0.9": 0.5817255784667957 + }, + "matched_mcc_signed_band": 0.5592740425946966, + "matched_mcc_abs_per_threshold": { + "0.6": 0.5403050901119101, + "0.7": 0.5519110181443877, + "0.8": 0.5616739199187185, + "0.9": 0.5823268972157295 + }, + "matched_mcc_abs_band": 0.5590542313476864, + "act_iou_mean": 0.08685330634218337, + "act_iou_std": 0.0019678935109149172, + "act_iou_n_pairs": 12288.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/dict_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x16/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..797dd78f0b9f62a29870a50b7ba1a04e945d2022 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8655, + "0.6": 6939, + "0.7": 5343, + "0.8": 3966, + "0.9": 3042 + }, + "n_dup_clusters_signed": { + "0.5": 2141, + "0.6": 1621, + "0.7": 1065, + "0.8": 562, + "0.9": 270 + }, + "largest_dup_signed": { + "0.5": 914, + "0.6": 832, + "0.7": 832, + "0.8": 678, + "0.9": 233 + }, + "babel_mu1_signed": 0.6928533911705017, + "n_dup_abs": { + "0.5": 8682, + "0.6": 6984, + "0.7": 5382, + "0.8": 3980, + "0.9": 3061 + }, + "n_dup_clusters_abs": { + "0.5": 2139, + "0.6": 1626, + "0.7": 1072, + "0.8": 551, + "0.9": 275 + }, + "largest_dup_abs": { + "0.5": 827, + "0.6": 803, + "0.7": 769, + "0.8": 684, + "0.9": 233 + }, + "babel_mu1_abs": 0.6942043304443359 + }, + "seed1": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8634, + "0.6": 6974, + "0.7": 5382, + "0.8": 3966, + "0.9": 2980 + }, + "n_dup_clusters_signed": { + "0.5": 2105, + "0.6": 1623, + "0.7": 1095, + "0.8": 585, + "0.9": 259 + }, + "largest_dup_signed": { + "0.5": 889, + "0.6": 889, + "0.7": 673, + "0.8": 508, + "0.9": 299 + }, + "babel_mu1_signed": 0.693641722202301, + "n_dup_abs": { + "0.5": 8670, + "0.6": 7031, + "0.7": 5423, + "0.8": 3995, + "0.9": 3003 + }, + "n_dup_clusters_abs": { + "0.5": 2104, + "0.6": 1629, + "0.7": 1095, + "0.8": 576, + "0.9": 260 + }, + "largest_dup_abs": { + "0.5": 1053, + "0.6": 805, + "0.7": 739, + "0.8": 529, + "0.9": 299 + }, + "babel_mu1_abs": 0.6951475143432617 + }, + "seed2": { + "d_sae": 12288, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 8649, + "0.6": 6884, + "0.7": 5257, + "0.8": 3951, + "0.9": 3023 + }, + "n_dup_clusters_signed": { + "0.5": 2181, + "0.6": 1619, + "0.7": 1043, + "0.8": 581, + "0.9": 292 + }, + "largest_dup_signed": { + "0.5": 701, + "0.6": 618, + "0.7": 501, + "0.8": 493, + "0.9": 214 + }, + "babel_mu1_signed": 0.6906366348266602, + "n_dup_abs": { + "0.5": 8673, + "0.6": 6933, + "0.7": 5290, + "0.8": 3965, + "0.9": 3044 + }, + "n_dup_clusters_abs": { + "0.5": 2178, + "0.6": 1629, + "0.7": 1043, + "0.8": 571, + "0.9": 287 + }, + "largest_dup_abs": { + "0.5": 822, + "0.6": 797, + "0.7": 797, + "0.8": 444, + "0.9": 214 + }, + "babel_mu1_abs": 0.6920421719551086 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8460356593132019, + "dcs_abs": 0.849884569644928, + "frac_sign_flipped": 0.0244140625, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8108665347099304, + "0.6": 0.8123555779457092, + "0.7": 0.8107208013534546, + "0.8": 0.8103269338607788, + "0.9": 0.8129523396492004 + }, + "unique_frac_signed": { + "0.5": 0.4692789713541667, + "0.6": 0.5658772786458334, + "0.7": 0.6514892578125, + "0.8": 0.7239176432291666, + "0.9": 0.7764892578125 + }, + "dedup_dcs_signed": { + "0.5": 0.38052261331419385, + "0.6": 0.4596935637406811, + "0.7": 0.5281758931669174, + "0.8": 0.5866099642056117, + "0.9": 0.6312487588511431 + }, + "matched_frac_signed": { + "0.5": 0.410400390625, + "0.6": 0.4790852864583333, + "0.7": 0.5068359375, + "0.8": 0.4720865885416667, + "0.9": 0.3621419270833333 + }, + "matched_cos_abs": { + "0.5": 0.8154089450836182, + "0.6": 0.8164052367210388, + "0.7": 0.8139873147010803, + "0.8": 0.8124246001243591, + "0.9": 0.815356969833374 + }, + "unique_frac_abs": { + "0.5": 0.4665934244791667, + "0.6": 0.5621744791666666, + "0.7": 0.6485188802083334, + "0.8": 0.7213541666666666, + "0.9": 0.7750244140625 + }, + "dedup_dcs_abs": { + "0.5": 0.38046445203751017, + "0.6": 0.45896218874258915, + "0.7": 0.5278861418337328, + "0.8": 0.5860458704022069, + "0.9": 0.6319215577968862 + }, + "matched_frac_abs": { + "0.5": 0.410888671875, + "0.6": 0.478759765625, + "0.7": 0.5084635416666666, + "0.8": 0.4715983072916667, + "0.9": 0.362060546875 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8714127540588379, + "dcs_abs": 0.8734491467475891, + "frac_sign_flipped": 0.015787760416666668, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8283454775810242, + "0.6": 0.8341121673583984, + "0.7": 0.8364167809486389, + "0.8": 0.8382395505905151, + "0.9": 0.8423116207122803 + }, + "unique_frac_signed": { + "0.5": 0.4717610677083333, + "0.6": 0.5693766276041666, + "0.7": 0.6544596354166666, + "0.8": 0.724365234375, + "0.9": 0.7760823567708334 + }, + "dedup_dcs_signed": { + "0.5": 0.3907811469349932, + "0.6": 0.47492397289412713, + "0.7": 0.5474010215160281, + "0.8": 0.6071915885258932, + "0.9": 0.6537031877378467 + }, + "matched_frac_signed": { + "0.5": 0.413330078125, + "0.6": 0.4869791666666667, + "0.7": 0.526123046875, + "0.8": 0.5148111979166666, + "0.9": 0.4423828125 + }, + "matched_cos_abs": { + "0.5": 0.8323161602020264, + "0.6": 0.8372353315353394, + "0.7": 0.8390799164772034, + "0.8": 0.8405229449272156, + "0.9": 0.843975841999054 + }, + "unique_frac_abs": { + "0.5": 0.469482421875, + "0.6": 0.566162109375, + "0.7": 0.6518147786458334, + "0.8": 0.7223714192708334, + "0.9": 0.7744547526041666 + }, + "dedup_dcs_abs": { + "0.5": 0.39075780665734783, + "0.6": 0.4740109213453252, + "0.7": 0.5469246900247526, + "0.8": 0.6071697526567732, + "0.9": 0.6536211019192706 + }, + "matched_frac_abs": { + "0.5": 0.4138997395833333, + "0.6": 0.488037109375, + "0.7": 0.52783203125, + "0.8": 0.51416015625, + "0.9": 0.44091796875 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.848410427570343, + "dcs_abs": 0.8519936203956604, + "frac_sign_flipped": 0.022542317708333332, + "alive_source": "stats", + "n_alive_a": 12288, + "n_alive_b": 12288, + "matched_cos_signed": { + "0.5": 0.8142401576042175, + "0.6": 0.817314088344574, + "0.7": 0.8153243064880371, + "0.8": 0.8121744394302368, + "0.9": 0.8141469359397888 + }, + "unique_frac_signed": { + "0.5": 0.4711507161458333, + "0.6": 0.5680338541666666, + "0.7": 0.6540934244791666, + "0.8": 0.7253011067708334, + "0.9": 0.7781575520833334 + }, + "dedup_dcs_signed": { + "0.5": 0.3836298333699233, + "0.6": 0.46426207166708383, + "0.7": 0.5332982676918618, + "0.8": 0.589071019809732, + "0.9": 0.6335345867070525 + }, + "matched_frac_signed": { + "0.5": 0.41259765625, + "0.6": 0.4830729166666667, + "0.7": 0.510986328125, + "0.8": 0.4785970052083333, + "0.9": 0.37451171875 + }, + "matched_cos_abs": { + "0.5": 0.818837583065033, + "0.6": 0.8209933042526245, + "0.7": 0.818362832069397, + "0.8": 0.8152377009391785, + "0.9": 0.8166646361351013 + }, + "unique_frac_abs": { + "0.5": 0.4685465494791667, + "0.6": 0.5643717447916666, + "0.7": 0.6510823567708334, + "0.8": 0.7227783203125, + "0.9": 0.7762044270833334 + }, + "dedup_dcs_abs": { + "0.5": 0.38366352412898175, + "0.6": 0.4633454235833293, + "0.7": 0.5328216013973968, + "0.8": 0.5892361361402436, + "0.9": 0.6338987060104653 + }, + "matched_frac_abs": { + "0.5": 0.4126790364583333, + "0.6": 0.4825032552083333, + "0.7": 0.51123046875, + "0.8": 0.4768880208333333, + "0.9": 0.3736165364583333 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.6923772493998209, + "babel_mu1_signed_std": 0.0015580984478361045, + "n_dup_signed_per_threshold": { + "0.5": 8646.0, + "0.6": 6932.333333333333, + "0.7": 5327.333333333333, + "0.8": 3961.0, + "0.9": 3015.0 + }, + "n_dup_signed_band": 4808.916666666666, + "n_dup_signed_band_std": 26.16812628625392, + "n_dup_clusters_signed_per_threshold": { + "0.5": 2142.3333333333335, + "0.6": 1621.0, + "0.7": 1067.6666666666667, + "0.8": 576.0, + "0.9": 273.6666666666667 + }, + "n_dup_clusters_signed_band": 884.5833333333334, + "n_dup_clusters_signed_band_std": 5.54714641354754, + "largest_dup_signed_per_threshold": { + "0.5": 834.6666666666666, + "0.6": 779.6666666666666, + "0.7": 668.6666666666666, + "0.8": 559.6666666666666, + "0.9": 248.66666666666666 + }, + "largest_dup_signed_band": 564.1666666666666, + "largest_dup_signed_band_std": 96.73234119638236, + "dcs_signed_mean": 0.8552862803141276, + "dcs_signed_std": 0.014016320760024861, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.38497786453970345, + "0.6": 0.4662932027672973, + "0.7": 0.5362917274582691, + "0.8": 0.5942908575137457, + "0.9": 0.6394955110986807 + }, + "dedup_dcs_signed_band": 0.5590928247094983, + "dedup_dcs_signed_band_std": 0.010302295727614024, + "matched_cos_signed_per_threshold": { + "0.5": 0.8178173899650574, + "0.6": 0.8212606112162272, + "0.7": 0.8208206295967102, + "0.8": 0.8202469746271769, + "0.9": 0.8231369654337565 + }, + "matched_cos_signed_band": 0.8213662952184676, + "matched_cos_signed_band_std": 0.014293149763982807, + "unique_frac_signed_per_threshold": { + "0.5": 0.4707302517361111, + "0.6": 0.5677625868055555, + "0.7": 0.653347439236111, + "0.8": 0.7245279947916666, + "0.9": 0.7769097222222223 + }, + "unique_frac_signed_band": 0.6806369357638888, + "unique_frac_signed_band_std": 0.0010464030773646793, + "matched_frac_signed_per_threshold": { + "0.5": 0.412109375, + "0.6": 0.4830457899305556, + "0.7": 0.5146484375, + "0.8": 0.48849826388888884, + "0.9": 0.39301215277777773 + }, + "matched_frac_signed_band": 0.4698011610243055, + "matched_frac_signed_band_std": 0.020008987227650564, + "babel_mu1_abs_mean": 0.6937980055809021, + "babel_mu1_abs_std": 0.001592046719759963, + "n_dup_abs_per_threshold": { + "0.5": 8675.0, + "0.6": 6982.666666666667, + "0.7": 5365.0, + "0.8": 3980.0, + "0.9": 3036.0 + }, + "n_dup_abs_band": 4840.916666666667, + "n_dup_abs_band_std": 29.056338952685234, + "n_dup_clusters_abs_per_threshold": { + "0.5": 2140.3333333333335, + "0.6": 1628.0, + "0.7": 1070.0, + "0.8": 566.0, + "0.9": 274.0 + }, + "n_dup_clusters_abs_band": 884.5, + "n_dup_clusters_abs_band_std": 4.8218253804964775, + "largest_dup_abs_per_threshold": { + "0.5": 900.6666666666666, + "0.6": 801.6666666666666, + "0.7": 768.3333333333334, + "0.8": 552.3333333333334, + "0.9": 248.66666666666666 + }, + "largest_dup_abs_band": 592.75, + "largest_dup_abs_band_std": 29.625791128677054, + "dcs_abs_mean": 0.8584424455960592, + "dcs_abs_std": 0.013038896937846736, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.38496192760794656, + "0.6": 0.4654395112237479, + "0.7": 0.5358774777519607, + "0.8": 0.5941505863997413, + "0.9": 0.6398137885755407 + }, + "dedup_dcs_abs_band": 0.5588203409877476, + "dedup_dcs_abs_band_std": 0.010217394660361043, + "matched_cos_abs_per_threshold": { + "0.5": 0.8221875627835592, + "0.6": 0.8248779575030009, + "0.7": 0.8238100210825602, + "0.8": 0.8227284153302511, + "0.9": 0.8253324826558431 + }, + "matched_cos_abs_band": 0.8241872191429138, + "matched_cos_abs_band_std": 0.013966608514550128, + "unique_frac_abs_per_threshold": { + "0.5": 0.46820746527777785, + "0.6": 0.564236111111111, + "0.7": 0.6504720052083334, + "0.8": 0.72216796875, + "0.9": 0.7752278645833334 + }, + "unique_frac_abs_band": 0.6780259874131944, + "unique_frac_abs_band_std": 0.0010904233032884723, + "matched_frac_abs_per_threshold": { + "0.5": 0.4124891493055555, + "0.6": 0.48310004340277773, + "0.7": 0.5158420138888888, + "0.8": 0.487548828125, + "0.9": 0.3921983506944444 + }, + "matched_frac_abs_band": 0.46967230902777773, + "matched_frac_abs_band_std": 0.02018668355975957, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/config.json b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..582c8dce741510337aae6bc33e8db9b021f25bc1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/lock.lock b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/model.safetensors b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..84541d8b4db74a7094607c2fa226cd119b681ef0 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca64feaf4a78eaac4082e45ec5f454e6a77b21c6cd3008d1ec9ef27dfa72ebe6 +size 75550008 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfb500a1e757ec803c90b616521553d5fb7a30a3 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63e55c4d2231e9f350ce2c654a615ff0fbfc44c3795a0bc082d7b0990161aa45 +size 99648 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7887da6d40a9f6a65e70e8a75e15ef67d99c8538 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3042e8b83d6f5ce39d9d63a8787ec95fba7b4801db6b426a86d47f09c993f3e7 +size 50471 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..987a3072d8adcf454d92de15604775947df65e9d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 32.03828061523438, + "l0_per_token_0.001": 32.03828061523438, + "l0_per_token_0.01": 32.03828061523438, + "l0_per_token_0.05": 32.03828061523438, + "l0_per_token_0.1": 32.03828061523438, + "l0_per_token_0.5": 31.459601025390626, + "l0_per_token_1.0": 15.88932890625, + "l1_per_token": 60.984214639496244, + "r2_score": 0.9197678115707759, + "l2_loss": 0.053416263354113445 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5151223a0a1f4237bca972f3ff4eefc033ca032a --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1c25daea99134583d3add9462a686544c76e3636d325f71725488870d274899 +size 50382 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfaf755af6248b5b802b85b9077dc1a85b09854c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2850e640a83d64efadc85dff2bccfa91642dfedbe253d24ab3b5fed9a5db34f +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ac2495e34c1689a790642aef613ac159fa2a1de --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da06c18f905e66ed53e9e7cae0de72ca5acc6501962f3c79c809b927341440e0 +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/config.json b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8465153a9e9a1f2031f87935b38abdc81b8f3232 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/lock.lock b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/model.safetensors b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a8d089cce87f1716e70fc858e70c44982f8e7186 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36b2b7bc941d2e08f3261ad9970fe72d4b2fda7326b285b9ecb8caaa20741234 +size 75550008 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..833e753ac7c30b9a1ab87b82f7f33e56877dd74d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a86a6850eacd5cf11024050560981076ce4e86dea2c9e88e37048ff98af4ff6 +size 99648 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..1651b125fc57b029b94b5d1eab8cfcdd27a62269 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3dac1be181bedf6465b34360e34bab47f270941332ae58ca2c9ee8bc789bcb1 +size 50471 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..8107d845e0d99e0dbac5aa99e41ff9af5013561c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 31.621306640625, + "l0_per_token_0.001": 31.621306640625, + "l0_per_token_0.01": 31.621306640625, + "l0_per_token_0.05": 31.621306640625, + "l0_per_token_0.1": 31.621306640625, + "l0_per_token_0.5": 31.088189208984375, + "l0_per_token_1.0": 15.836713134765626, + "l1_per_token": 60.57542369279035, + "r2_score": 0.9198098326599718, + "l2_loss": 0.05338828693204572 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..aca19f76a133632c86cbdd2cf0b173270a918e78 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8f265eb2ebaa7f69592fe7ccf1b913bbd4e0c9c9b64ca28bc1e6720edbae952 +size 50382 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1db008aead1355c41a831a13c432e98cf198cdd --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:019c4a5720c15455f35641c079925d8136e1d5bac61785ca7c5da29d32fff851 +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5290e110750731bbcde2658c4a8d1aba2a7335b --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d9139e352cdf2cb3d40a02d4cd8f2af13ec356b63580c63e754913b3057433b +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/config.json b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..859d3bae315c9ef5dc1a4ece37711d5e66c114f1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 16, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 1228, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/lock.lock b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/lock.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/model.safetensors b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..00d2048bcffd6adb83cf3e50c2379e2ab52fef82 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:062c936841ac8e0b34721eec7f7cc9a9faa05dbd11fae1007a18f9581c94c14d +size 75550008 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..096b298cfc1101a0819b6446517045a069aa23f0 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bffb088ed9312df38e642f926da185f2b79cbe04455e4b84dda59ac6d9b33f0d +size 99648 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2b7e6aa0e8d9d16604813d323708c250215176d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4a961aa9c3193c93db12860e4f44dd13d8897e5df32d2f43b3752d1f5360607 +size 50471 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e175dff7ff72e607cfdfe8ee787e9ed0c23d93c2 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 30.96504931640625, + "l0_per_token_0.001": 30.96504931640625, + "l0_per_token_0.01": 30.96504931640625, + "l0_per_token_0.05": 30.96504931640625, + "l0_per_token_0.1": 30.96504931640625, + "l0_per_token_0.5": 30.4920673828125, + "l0_per_token_1.0": 15.85408134765625, + "l1_per_token": 60.35047452306022, + "r2_score": 0.9189396134275296, + "l2_loss": 0.05396765365014229 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..28b278b220ba1cbf39f3bc18dec1e933911544bc --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b16d9bf8db2727909803ab3f997fa529513a1418084f126603313133b63a7c33 +size 50382 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..09c87dd8c3487e5807cb2c755845a011e6e99657 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c8a03084d5b49df589da4d776b121abd6a13a5abcd0d5a1748be6a38862ba73 +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac90e1883589da6c300eefe3cbf3fd5d4044d4c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 12288, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dced1c9e5c906272c657860bcc8c235979171f7e --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c443d9a1c2a39b93121def1f1f210068194a97fa3e7fe94c8d0b59b95aaeed8 +size 50387 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/code_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x4/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..374f9395534178d36fea59df39a576d251849794 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 74.59344666666667, + "recon_r2": 0.9300341418178543, + "l1_per_token": 90.61562497306788 + }, + "seed1": { + "l0": 74.47944833333334, + "recon_r2": 0.9298792853295661, + "l1_per_token": 90.80396640408732 + }, + "seed2": { + "l0": 74.52445833333333, + "recon_r2": 0.9299858808227701, + "l1_per_token": 90.96650383977766 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.5095629276037345, + "mcc_abs": 0.50958084033195, + "act_iou": 0.18173930444543884, + "act_iou_n_pairs": 3072, + "input_r2": 0.9591212125781509, + "matched_mcc_signed": { + "0.6": 0.6751159937716986, + "0.7": 0.6877317280110115, + "0.8": 0.7060827731892402, + "0.9": 0.7322452661579198 + }, + "matched_mcc_abs": { + "0.6": 0.6736513309495497, + "0.7": 0.6857773057017217, + "0.8": 0.7057637926598147, + "0.9": 0.7352221765968884 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5077228670479319, + "mcc_abs": 0.5077579783694471, + "act_iou": 0.18160556076511405, + "act_iou_n_pairs": 3072, + "input_r2": 0.9595337169869665, + "matched_mcc_signed": { + "0.6": 0.6688639914284199, + "0.7": 0.6766782830651406, + "0.8": 0.6922198067221182, + "0.9": 0.7261391065480411 + }, + "matched_mcc_abs": { + "0.6": 0.6660090368323724, + "0.7": 0.6748041807175686, + "0.8": 0.691154152679693, + "0.9": 0.7240905033690975 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5104842859544187, + "mcc_abs": 0.5105389665327583, + "act_iou": 0.18278688156784595, + "act_iou_n_pairs": 3072, + "input_r2": 0.9594291308051673, + "matched_mcc_signed": { + "0.6": 0.6709192438419083, + "0.7": 0.6676986098268255, + "0.8": 0.6865462476668585, + "0.9": 0.7166088647590287 + }, + "matched_mcc_abs": { + "0.6": 0.6690339678029686, + "0.7": 0.6700127117925008, + "0.8": 0.6862882359230246, + "0.9": 0.7178945609492063 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 74.53245111111111, + "l0_std": 0.05741792728014375, + "recon_r2_mean": 0.9299664359900635, + "recon_r2_std": 7.923830595411145e-05, + "l1_per_token_mean": 90.79536507231096, + "l1_per_token_std": 0.17559749986013412, + "input_r2_mean": 0.9593613534567615, + "input_r2_std": 0.0002144418302029267, + "mcc_signed_mean": 0.5092566935353617, + "mcc_signed_std": 0.0014059491714673918, + "mcc_abs_mean": 0.5092925950780517, + "mcc_abs_std": 0.0014127235348486732, + "matched_mcc_signed_per_threshold": { + "0.6": 0.6716330763473423, + "0.7": 0.6773695403009925, + "0.8": 0.6949496091927388, + "0.9": 0.7249977458216632 + }, + "matched_mcc_signed_band": 0.6922374929156841, + "matched_mcc_abs_per_threshold": { + "0.6": 0.6695647785282969, + "0.7": 0.6768647327372638, + "0.8": 0.6944020604208441, + "0.9": 0.7257357469717308 + }, + "matched_mcc_abs_band": 0.6916418296645339, + "act_iou_mean": 0.18204391559279962, + "act_iou_std": 0.0006468930924535495, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/dict_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x4/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..527d0262e8dd62abb68a8ba572757a20309e4e75 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1523, + "0.6": 1214, + "0.7": 970, + "0.8": 748, + "0.9": 559 + }, + "n_dup_clusters_signed": { + "0.5": 373, + "0.6": 268, + "0.7": 192, + "0.8": 124, + "0.9": 75 + }, + "n_dup_removed_signed": { + "0.5": 1150, + "0.6": 946, + "0.7": 778, + "0.8": 624, + "0.9": 484 + }, + "largest_dup_signed": { + "0.5": 199, + "0.6": 199, + "0.7": 101, + "0.8": 101, + "0.9": 61 + }, + "babel_mu1_signed": 0.5888772010803223, + "n_dup_abs": { + "0.5": 1565, + "0.6": 1261, + "0.7": 1001, + "0.8": 775, + "0.9": 563 + }, + "n_dup_clusters_abs": { + "0.5": 387, + "0.6": 288, + "0.7": 201, + "0.8": 130, + "0.9": 75 + }, + "n_dup_removed_abs": { + "0.5": 1178, + "0.6": 973, + "0.7": 800, + "0.8": 645, + "0.9": 488 + }, + "largest_dup_abs": { + "0.5": 219, + "0.6": 219, + "0.7": 196, + "0.8": 108, + "0.9": 68 + }, + "babel_mu1_abs": 0.5962309241294861, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1523, + "0.6": 1217, + "0.7": 943, + "0.8": 739, + "0.9": 569 + }, + "n_dup_clusters_signed": { + "0.5": 369, + "0.6": 265, + "0.7": 175, + "0.8": 115, + "0.9": 75 + }, + "n_dup_removed_signed": { + "0.5": 1154, + "0.6": 952, + "0.7": 768, + "0.8": 624, + "0.9": 494 + }, + "largest_dup_signed": { + "0.5": 131, + "0.6": 131, + "0.7": 122, + "0.8": 98, + "0.9": 90 + }, + "babel_mu1_signed": 0.5888288021087646, + "n_dup_abs": { + "0.5": 1562, + "0.6": 1257, + "0.7": 971, + "0.8": 757, + "0.9": 585 + }, + "n_dup_clusters_abs": { + "0.5": 382, + "0.6": 278, + "0.7": 183, + "0.8": 118, + "0.9": 76 + }, + "n_dup_removed_abs": { + "0.5": 1180, + "0.6": 979, + "0.7": 788, + "0.8": 639, + "0.9": 509 + }, + "largest_dup_abs": { + "0.5": 165, + "0.6": 165, + "0.7": 165, + "0.8": 142, + "0.9": 60 + }, + "babel_mu1_abs": 0.595833957195282, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1558, + "0.6": 1234, + "0.7": 975, + "0.8": 750, + "0.9": 565 + }, + "n_dup_clusters_signed": { + "0.5": 392, + "0.6": 278, + "0.7": 195, + "0.8": 127, + "0.9": 80 + }, + "n_dup_removed_signed": { + "0.5": 1166, + "0.6": 956, + "0.7": 780, + "0.8": 623, + "0.9": 485 + }, + "largest_dup_signed": { + "0.5": 164, + "0.6": 164, + "0.7": 164, + "0.8": 154, + "0.9": 69 + }, + "babel_mu1_signed": 0.5905370712280273, + "n_dup_abs": { + "0.5": 1590, + "0.6": 1272, + "0.7": 998, + "0.8": 762, + "0.9": 581 + }, + "n_dup_clusters_abs": { + "0.5": 396, + "0.6": 288, + "0.7": 198, + "0.8": 123, + "0.9": 83 + }, + "n_dup_removed_abs": { + "0.5": 1194, + "0.6": 984, + "0.7": 800, + "0.8": 639, + "0.9": 498 + }, + "largest_dup_abs": { + "0.5": 174, + "0.6": 174, + "0.7": 174, + "0.8": 97, + "0.9": 83 + }, + "babel_mu1_abs": 0.5972307324409485, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8212807774543762, + "dcs_abs": 0.8252390027046204, + "frac_sign_flipped": 0.037434895833333336, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7926982045173645, + "0.6": 0.7928213477134705, + "0.7": 0.7910512685775757, + "0.8": 0.7900389432907104, + "0.9": 0.7926890254020691 + }, + "unique_frac_signed": { + "0.5": 0.625, + "0.6": 0.6910807291666666, + "0.7": 0.7483723958333334, + "0.8": 0.796875, + "0.9": 0.8408203125 + }, + "dedup_dcs_signed": { + "0.5": 0.4954363778233528, + "0.6": 0.5479035550767245, + "0.7": 0.592000933092398, + "0.8": 0.6295622829347849, + "0.9": 0.6665090340538882 + }, + "matched_frac_signed": { + "0.5": 0.5465494791666666, + "0.6": 0.57421875, + "0.7": 0.5634765625, + "0.8": 0.5026041666666666, + "0.9": 0.3414713541666667 + }, + "matched_cos_abs": { + "0.5": 0.7979190349578857, + "0.6": 0.7973155975341797, + "0.7": 0.7959036827087402, + "0.8": 0.7952700257301331, + "0.9": 0.7975999712944031 + }, + "unique_frac_abs": { + "0.5": 0.6162109375, + "0.6": 0.6822916666666666, + "0.7": 0.7415364583333334, + "0.8": 0.791015625, + "0.9": 0.8377278645833334 + }, + "dedup_dcs_abs": { + "0.5": 0.49168643658049405, + "0.6": 0.5440017879009247, + "0.7": 0.5901915980502963, + "0.8": 0.6290710164466873, + "0.9": 0.6681717207441883 + }, + "matched_frac_abs": { + "0.5": 0.5406901041666666, + "0.6": 0.5738932291666666, + "0.7": 0.5651041666666666, + "0.8": 0.4996744791666667, + "0.9": 0.3388671875 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8383877873420715, + "dcs_abs": 0.8434087634086609, + "frac_sign_flipped": 0.030598958333333332, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8179414868354797, + "0.6": 0.8152375817298889, + "0.7": 0.8111293911933899, + "0.8": 0.8126823902130127, + "0.9": 0.8143463134765625 + }, + "unique_frac_signed": { + "0.5": 0.623046875, + "0.6": 0.6904296875, + "0.7": 0.7464192708333334, + "0.8": 0.7970377604166666, + "0.9": 0.84228515625 + }, + "dedup_dcs_signed": { + "0.5": 0.5096158873056993, + "0.6": 0.5628642287920229, + "0.7": 0.6054426087260557, + "0.8": 0.6477385522254432, + "0.9": 0.6859118118882179 + }, + "matched_frac_signed": { + "0.5": 0.5524088541666666, + "0.6": 0.5882161458333334, + "0.7": 0.5862630208333334, + "0.8": 0.544921875, + "0.9": 0.4075520833333333 + }, + "matched_cos_abs": { + "0.5": 0.82192063331604, + "0.6": 0.8198916912078857, + "0.7": 0.8148212432861328, + "0.8": 0.816645085811615, + "0.9": 0.8187885880470276 + }, + "unique_frac_abs": { + "0.5": 0.6139322916666666, + "0.6": 0.6814778645833334, + "0.7": 0.7395833333333334, + "0.8": 0.791015625, + "0.9": 0.8395182291666666 + }, + "dedup_dcs_abs": { + "0.5": 0.5046036179798344, + "0.6": 0.5587380389139678, + "0.7": 0.6026282111803691, + "0.8": 0.6459790229564533, + "0.9": 0.6873879454991159 + }, + "matched_frac_abs": { + "0.5": 0.5478515625, + "0.6": 0.5869140625, + "0.7": 0.5862630208333334, + "0.8": 0.5439453125, + "0.9": 0.4075520833333333 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8456637263298035, + "dcs_abs": 0.8502656817436218, + "frac_sign_flipped": 0.031575520833333336, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8194010257720947, + "0.6": 0.8174140453338623, + "0.7": 0.8198081851005554, + "0.8": 0.8196272253990173, + "0.9": 0.8235411643981934 + }, + "unique_frac_signed": { + "0.5": 0.6223958333333334, + "0.6": 0.689453125, + "0.7": 0.748046875, + "0.8": 0.7970377604166666, + "0.9": 0.8406575520833334 + }, + "dedup_dcs_signed": { + "0.5": 0.5099917842696111, + "0.6": 0.563568667974323, + "0.7": 0.613254950963892, + "0.8": 0.6532738481085592, + "0.9": 0.6923160993028432 + }, + "matched_frac_signed": { + "0.5": 0.552734375, + "0.6": 0.5833333333333334, + "0.7": 0.5934244791666666, + "0.8": 0.5517578125, + "0.9": 0.4326171875 + }, + "matched_cos_abs": { + "0.5": 0.8246318697929382, + "0.6": 0.8222889304161072, + "0.7": 0.823699951171875, + "0.8": 0.8227087259292603, + "0.9": 0.8263576030731201 + }, + "unique_frac_abs": { + "0.5": 0.6136067708333334, + "0.6": 0.6805013020833334, + "0.7": 0.7415364583333334, + "0.8": 0.7919921875, + "0.9": 0.8361002604166666 + }, + "dedup_dcs_abs": { + "0.5": 0.5059996987498987, + "0.6": 0.5595686878368724, + "0.7": 0.6108035445213318, + "0.8": 0.6515788835240528, + "0.9": 0.6909178071267281 + }, + "matched_frac_abs": { + "0.5": 0.5475260416666666, + "0.6": 0.5826822916666666, + "0.7": 0.5934244791666666, + "0.8": 0.5517578125, + "0.9": 0.4296875 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.5894143581390381, + "babel_mu1_signed_std": 0.000972599159599066, + "n_dup_signed_per_threshold": { + "0.5": 1534.6666666666667, + "0.6": 1221.6666666666667, + "0.7": 962.6666666666666, + "0.8": 745.6666666666666, + "0.9": 564.3333333333334 + }, + "n_dup_signed_band": 873.5833333333334, + "n_dup_signed_band_std": 7.0371040445152815, + "n_dup_clusters_signed_per_threshold": { + "0.5": 378.0, + "0.6": 270.3333333333333, + "0.7": 187.33333333333334, + "0.8": 122.0, + "0.9": 76.66666666666667 + }, + "n_dup_clusters_signed_band": 164.08333333333331, + "n_dup_clusters_signed_band_std": 6.276610019216848, + "largest_dup_signed_per_threshold": { + "0.5": 164.66666666666666, + "0.6": 164.66666666666666, + "0.7": 129.0, + "0.8": 117.66666666666667, + "0.9": 73.33333333333333 + }, + "largest_dup_signed_band": 121.16666666666666, + "largest_dup_signed_band_std": 14.59951483212142, + "dcs_signed_mean": 0.8351107637087504, + "dcs_signed_std": 0.01251743631441493, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5050146831328878, + "0.6": 0.5581121506143568, + "0.7": 0.6035661642607819, + "0.8": 0.643524894422929, + "0.9": 0.6815789817483164 + }, + "dedup_dcs_signed_band": 0.621695547761596, + "dedup_dcs_signed_band_std": 0.011293201355414123, + "matched_cos_signed_per_threshold": { + "0.5": 0.8100135723749796, + "0.6": 0.8084909915924072, + "0.7": 0.8073296149571737, + "0.8": 0.8074495196342468, + "0.9": 0.8101921677589417 + }, + "matched_cos_signed_band": 0.8083655734856924, + "matched_cos_signed_band_std": 0.014864067072406676, + "unique_frac_signed_per_threshold": { + "0.5": 0.6234809027777778, + "0.6": 0.6903211805555555, + "0.7": 0.7476128472222223, + "0.8": 0.7969835069444443, + "0.9": 0.8412543402777778 + }, + "unique_frac_signed_band": 0.7690429687499999, + "unique_frac_signed_band_std": 0.000244140625, + "matched_frac_signed_per_threshold": { + "0.5": 0.550564236111111, + "0.6": 0.5819227430555557, + "0.7": 0.5810546875, + "0.8": 0.5330946180555555, + "0.9": 0.3938802083333333 + }, + "matched_frac_signed_band": 0.5224880642361112, + "matched_frac_signed_band_std": 0.0238084518478922, + "babel_mu1_abs_mean": 0.5964318712552389, + "babel_mu1_abs_std": 0.0007197430668216988, + "n_dup_abs_per_threshold": { + "0.5": 1572.3333333333333, + "0.6": 1263.3333333333333, + "0.7": 990.0, + "0.8": 764.6666666666666, + "0.9": 576.3333333333334 + }, + "n_dup_abs_band": 898.5833333333333, + "n_dup_abs_band_std": 5.513241635674364, + "n_dup_clusters_abs_per_threshold": { + "0.5": 388.3333333333333, + "0.6": 284.6666666666667, + "0.7": 194.0, + "0.8": 123.66666666666667, + "0.9": 78.0 + }, + "n_dup_clusters_abs_band": 170.08333333333334, + "n_dup_clusters_abs_band_std": 5.490522136676377, + "largest_dup_abs_per_threshold": { + "0.5": 186.0, + "0.6": 186.0, + "0.7": 178.33333333333334, + "0.8": 115.66666666666667, + "0.9": 70.33333333333333 + }, + "largest_dup_abs_band": 137.58333333333334, + "largest_dup_abs_band_std": 8.818777315100622, + "dcs_abs_mean": 0.839637815952301, + "dcs_abs_std": 0.012932466874856824, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5007632511034091, + "0.6": 0.5541028382172549, + "0.7": 0.601207784583999, + "0.8": 0.6422096409757311, + "0.9": 0.6821591577900108 + }, + "dedup_dcs_abs_band": 0.619919855391749, + "dedup_dcs_abs_band_std": 0.010688158828354681, + "matched_cos_abs_per_threshold": { + "0.5": 0.814823846022288, + "0.6": 0.8131654063860575, + "0.7": 0.8114749590555826, + "0.8": 0.8115412791570028, + "0.9": 0.8142487208048502 + }, + "matched_cos_abs_band": 0.8126075913508732, + "matched_cos_abs_band_std": 0.014273973985141597, + "unique_frac_abs_per_threshold": { + "0.5": 0.6145833333333334, + "0.6": 0.6814236111111112, + "0.7": 0.7408854166666666, + "0.8": 0.7913411458333334, + "0.9": 0.8377821180555555 + }, + "unique_frac_abs_band": 0.7628580729166666, + "unique_frac_abs_band_std": 0.00030720354961225385, + "matched_frac_abs_per_threshold": { + "0.5": 0.5453559027777777, + "0.6": 0.5811631944444443, + "0.7": 0.5815972222222222, + "0.8": 0.5317925347222222, + "0.9": 0.39203559027777773 + }, + "matched_frac_abs_band": 0.5216471354166666, + "matched_frac_abs_band_std": 0.02396491692796204, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 3072 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/config.json b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e29bc53bb30892fe2d225d5ad62f012886441434 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 32, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 32, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/model.safetensors b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b861996a69c314513ca276ad9b5dcd6a7df7d72f --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:571b80554e326186ad6cb1e6dd97ccd1077cd84889792108724c9a7422394c9e +size 18890032 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..06f8989dc94866076d7e98ef22333663a644ea54 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e982a82de9809f633de4fd56f6df1c53e313dbecefcbe678283ef3e1e3cf155 +size 25920 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..96227314759ef0055b546e776c0aa75436707ee8 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e99e11d35ca557f41c978a4a1b14e55733f696346a2478994990563efc4902ca +size 13607 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..31754bbf4121e2cd8ac2625e98f0cc15a9f707a9 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 11.807441845703124, + "l0_per_token_0.001": 11.807441845703124, + "l0_per_token_0.01": 11.807441845703124, + "l0_per_token_0.05": 11.807441845703124, + "l0_per_token_0.1": 11.807441845703124, + "l0_per_token_0.5": 11.8061119140625, + "l0_per_token_1.0": 11.75499580078125, + "l1_per_token": 44.71890905806031, + "r2_score": 0.8547333848294265, + "l2_loss": 0.09671429789500322 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..52ab11b4f323e38f175ebe8db931494909b31778 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d34e8f3c0a8c8925b126b8f42b13aa698e82bf8bfe7b15d0bf0681f15254844 +size 13518 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d678d3558d06d9563e7a5aed6967bfcfaa53d22f --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b18011d1cf121ff1f46329097f669e52640ac7391966ffd1f962ab5d2973ec8 +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6808967ab2c6ff1a5ca9bdb102393c66cbe3f7c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3684e2f12e6def98502a0b47ab2ee05cb3f3696e95f3b7029d41402343e66edc +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/config.json b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d39c416079930c68d7c93074e3b2b5646a514bb7 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 32, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 32, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/model.safetensors b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..95bf365c4211956f6cdbbee0803ab6e95ce932fc --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6566abdca75216b0030458c73cec7cb6320d013a3a498849a12b99112563f26a +size 18890032 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..93c35fd9e60b50670dc6a4d3df6aa2743d2846a5 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0866001df5e56e48203c3a60265a86fe4e842d969f6064d843cbae1b2eae9006 +size 25920 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..e14d94f220ea90370b93a5305adafd20761d60c6 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc8b8da5da5c6baf9efa1a657224294a61eefd2ec67219c3b1ba283519c3141b +size 13607 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..900307cdae545519e7c70736d42fa708052c44ad --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 12.081834521484375, + "l0_per_token_0.001": 12.081834521484375, + "l0_per_token_0.01": 12.081834521484375, + "l0_per_token_0.05": 12.081834521484375, + "l0_per_token_0.1": 12.081834521484375, + "l0_per_token_0.5": 12.081421044921875, + "l0_per_token_1.0": 12.020004345703125, + "l1_per_token": 45.32236212879355, + "r2_score": 0.8558574905656944, + "l2_loss": 0.09596590090843317 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..56eb2b7a0ced3d33ec7d1f24223c15ea73839711 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c103ed13fa45732ffbab83ab5754b6f6634918afb9930e4e70ffa839e9a6794f +size 13518 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd3c8662d57fb5e7bd3474b4caf6064251153592 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf4137fee1e012d6badfbebfe1095f132a1ca2f096b58771eaeeb5ca6f41bf26 +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fb28f8da2fadc95e15057edf1721af541b70c50 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:734882fe8dd56e1bd168cc69e1fba5945c534e2a08fd6aa60f2ba2ccf78dc223 +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/config.json b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbfe8b7905bce01e56935c8b4e6059bb883ce1f9 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 32, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 32, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/model.safetensors b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ad61fd48c1106d8228628eebdf6a0bad6ea9c88c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87740febb2cca29e4547f955f3b3634ef6aff8549ae123bde46b9f38df2948dc +size 18890032 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..07674f31c203a1db31e0af72712d7f5936fecfd6 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e898faf545aaa007d6f2827990fa9f79db066a023568887359f0c6dcdfc087d +size 25920 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa2f1e1a5535b1ed6f6c15347d75b853ef959db5 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b423798602411b4fe1c0b80de177fc7f6eefc23ebf3d4dcde13217d7202226f +size 13607 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..97a21616b33502a9bef42d8dcd5358c846804595 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 12.7585103515625, + "l0_per_token_0.001": 12.7585103515625, + "l0_per_token_0.01": 12.7585103515625, + "l0_per_token_0.05": 12.7585103515625, + "l0_per_token_0.1": 12.7585103515625, + "l0_per_token_0.5": 12.757329052734375, + "l0_per_token_1.0": 12.665068115234375, + "l1_per_token": 46.4482664841365, + "r2_score": 0.8583668927531972, + "l2_loss": 0.09429521373495188 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8433ebab4b42652cddf1b492eccabb1efe1145e1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1875ffbab06df3d9c8543d48322c3d0f4a3b7093645a3dcaa9bbd8de16751e36 +size 13518 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..611e31b1d8465a1131e82a9a33fd1007340fedce --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:508c3e59c421c46ed74596335024eadc0ae2190b553da82d4e7b874c1be08966 +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4da825740f41cea60ce172c919380a2512f1fa5 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:536eea5d44e26037d9617baa915937daca6081504969f53d5420f68ef0f7a57c +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/code_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x4/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..374f9395534178d36fea59df39a576d251849794 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 74.59344666666667, + "recon_r2": 0.9300341418178543, + "l1_per_token": 90.61562497306788 + }, + "seed1": { + "l0": 74.47944833333334, + "recon_r2": 0.9298792853295661, + "l1_per_token": 90.80396640408732 + }, + "seed2": { + "l0": 74.52445833333333, + "recon_r2": 0.9299858808227701, + "l1_per_token": 90.96650383977766 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.5095629276037345, + "mcc_abs": 0.50958084033195, + "act_iou": 0.18173930444543884, + "act_iou_n_pairs": 3072, + "input_r2": 0.9591212125781509, + "matched_mcc_signed": { + "0.6": 0.6751159937716986, + "0.7": 0.6877317280110115, + "0.8": 0.7060827731892402, + "0.9": 0.7322452661579198 + }, + "matched_mcc_abs": { + "0.6": 0.6736513309495497, + "0.7": 0.6857773057017217, + "0.8": 0.7057637926598147, + "0.9": 0.7352221765968884 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5077228670479319, + "mcc_abs": 0.5077579783694471, + "act_iou": 0.18160556076511405, + "act_iou_n_pairs": 3072, + "input_r2": 0.9595337169869665, + "matched_mcc_signed": { + "0.6": 0.6688639914284199, + "0.7": 0.6766782830651406, + "0.8": 0.6922198067221182, + "0.9": 0.7261391065480411 + }, + "matched_mcc_abs": { + "0.6": 0.6660090368323724, + "0.7": 0.6748041807175686, + "0.8": 0.691154152679693, + "0.9": 0.7240905033690975 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5104842859544187, + "mcc_abs": 0.5105389665327583, + "act_iou": 0.18278688156784595, + "act_iou_n_pairs": 3072, + "input_r2": 0.9594291308051673, + "matched_mcc_signed": { + "0.6": 0.6709192438419083, + "0.7": 0.6676986098268255, + "0.8": 0.6865462476668585, + "0.9": 0.7166088647590287 + }, + "matched_mcc_abs": { + "0.6": 0.6690339678029686, + "0.7": 0.6700127117925008, + "0.8": 0.6862882359230246, + "0.9": 0.7178945609492063 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 74.53245111111111, + "l0_std": 0.05741792728014375, + "recon_r2_mean": 0.9299664359900635, + "recon_r2_std": 7.923830595411145e-05, + "l1_per_token_mean": 90.79536507231096, + "l1_per_token_std": 0.17559749986013412, + "input_r2_mean": 0.9593613534567615, + "input_r2_std": 0.0002144418302029267, + "mcc_signed_mean": 0.5092566935353617, + "mcc_signed_std": 0.0014059491714673918, + "mcc_abs_mean": 0.5092925950780517, + "mcc_abs_std": 0.0014127235348486732, + "matched_mcc_signed_per_threshold": { + "0.6": 0.6716330763473423, + "0.7": 0.6773695403009925, + "0.8": 0.6949496091927388, + "0.9": 0.7249977458216632 + }, + "matched_mcc_signed_band": 0.6922374929156841, + "matched_mcc_abs_per_threshold": { + "0.6": 0.6695647785282969, + "0.7": 0.6768647327372638, + "0.8": 0.6944020604208441, + "0.9": 0.7257357469717308 + }, + "matched_mcc_abs_band": 0.6916418296645339, + "act_iou_mean": 0.18204391559279962, + "act_iou_std": 0.0006468930924535495, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/dict_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x4/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..527d0262e8dd62abb68a8ba572757a20309e4e75 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1523, + "0.6": 1214, + "0.7": 970, + "0.8": 748, + "0.9": 559 + }, + "n_dup_clusters_signed": { + "0.5": 373, + "0.6": 268, + "0.7": 192, + "0.8": 124, + "0.9": 75 + }, + "n_dup_removed_signed": { + "0.5": 1150, + "0.6": 946, + "0.7": 778, + "0.8": 624, + "0.9": 484 + }, + "largest_dup_signed": { + "0.5": 199, + "0.6": 199, + "0.7": 101, + "0.8": 101, + "0.9": 61 + }, + "babel_mu1_signed": 0.5888772010803223, + "n_dup_abs": { + "0.5": 1565, + "0.6": 1261, + "0.7": 1001, + "0.8": 775, + "0.9": 563 + }, + "n_dup_clusters_abs": { + "0.5": 387, + "0.6": 288, + "0.7": 201, + "0.8": 130, + "0.9": 75 + }, + "n_dup_removed_abs": { + "0.5": 1178, + "0.6": 973, + "0.7": 800, + "0.8": 645, + "0.9": 488 + }, + "largest_dup_abs": { + "0.5": 219, + "0.6": 219, + "0.7": 196, + "0.8": 108, + "0.9": 68 + }, + "babel_mu1_abs": 0.5962309241294861, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1523, + "0.6": 1217, + "0.7": 943, + "0.8": 739, + "0.9": 569 + }, + "n_dup_clusters_signed": { + "0.5": 369, + "0.6": 265, + "0.7": 175, + "0.8": 115, + "0.9": 75 + }, + "n_dup_removed_signed": { + "0.5": 1154, + "0.6": 952, + "0.7": 768, + "0.8": 624, + "0.9": 494 + }, + "largest_dup_signed": { + "0.5": 131, + "0.6": 131, + "0.7": 122, + "0.8": 98, + "0.9": 90 + }, + "babel_mu1_signed": 0.5888288021087646, + "n_dup_abs": { + "0.5": 1562, + "0.6": 1257, + "0.7": 971, + "0.8": 757, + "0.9": 585 + }, + "n_dup_clusters_abs": { + "0.5": 382, + "0.6": 278, + "0.7": 183, + "0.8": 118, + "0.9": 76 + }, + "n_dup_removed_abs": { + "0.5": 1180, + "0.6": 979, + "0.7": 788, + "0.8": 639, + "0.9": 509 + }, + "largest_dup_abs": { + "0.5": 165, + "0.6": 165, + "0.7": 165, + "0.8": 142, + "0.9": 60 + }, + "babel_mu1_abs": 0.595833957195282, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1558, + "0.6": 1234, + "0.7": 975, + "0.8": 750, + "0.9": 565 + }, + "n_dup_clusters_signed": { + "0.5": 392, + "0.6": 278, + "0.7": 195, + "0.8": 127, + "0.9": 80 + }, + "n_dup_removed_signed": { + "0.5": 1166, + "0.6": 956, + "0.7": 780, + "0.8": 623, + "0.9": 485 + }, + "largest_dup_signed": { + "0.5": 164, + "0.6": 164, + "0.7": 164, + "0.8": 154, + "0.9": 69 + }, + "babel_mu1_signed": 0.5905370712280273, + "n_dup_abs": { + "0.5": 1590, + "0.6": 1272, + "0.7": 998, + "0.8": 762, + "0.9": 581 + }, + "n_dup_clusters_abs": { + "0.5": 396, + "0.6": 288, + "0.7": 198, + "0.8": 123, + "0.9": 83 + }, + "n_dup_removed_abs": { + "0.5": 1194, + "0.6": 984, + "0.7": 800, + "0.8": 639, + "0.9": 498 + }, + "largest_dup_abs": { + "0.5": 174, + "0.6": 174, + "0.7": 174, + "0.8": 97, + "0.9": 83 + }, + "babel_mu1_abs": 0.5972307324409485, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8212807774543762, + "dcs_abs": 0.8252390027046204, + "frac_sign_flipped": 0.037434895833333336, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7926982045173645, + "0.6": 0.7928213477134705, + "0.7": 0.7910512685775757, + "0.8": 0.7900389432907104, + "0.9": 0.7926890254020691 + }, + "unique_frac_signed": { + "0.5": 0.625, + "0.6": 0.6910807291666666, + "0.7": 0.7483723958333334, + "0.8": 0.796875, + "0.9": 0.8408203125 + }, + "dedup_dcs_signed": { + "0.5": 0.4954363778233528, + "0.6": 0.5479035550767245, + "0.7": 0.592000933092398, + "0.8": 0.6295622829347849, + "0.9": 0.6665090340538882 + }, + "matched_frac_signed": { + "0.5": 0.5465494791666666, + "0.6": 0.57421875, + "0.7": 0.5634765625, + "0.8": 0.5026041666666666, + "0.9": 0.3414713541666667 + }, + "matched_cos_abs": { + "0.5": 0.7979190349578857, + "0.6": 0.7973155975341797, + "0.7": 0.7959036827087402, + "0.8": 0.7952700257301331, + "0.9": 0.7975999712944031 + }, + "unique_frac_abs": { + "0.5": 0.6162109375, + "0.6": 0.6822916666666666, + "0.7": 0.7415364583333334, + "0.8": 0.791015625, + "0.9": 0.8377278645833334 + }, + "dedup_dcs_abs": { + "0.5": 0.49168643658049405, + "0.6": 0.5440017879009247, + "0.7": 0.5901915980502963, + "0.8": 0.6290710164466873, + "0.9": 0.6681717207441883 + }, + "matched_frac_abs": { + "0.5": 0.5406901041666666, + "0.6": 0.5738932291666666, + "0.7": 0.5651041666666666, + "0.8": 0.4996744791666667, + "0.9": 0.3388671875 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8383877873420715, + "dcs_abs": 0.8434087634086609, + "frac_sign_flipped": 0.030598958333333332, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8179414868354797, + "0.6": 0.8152375817298889, + "0.7": 0.8111293911933899, + "0.8": 0.8126823902130127, + "0.9": 0.8143463134765625 + }, + "unique_frac_signed": { + "0.5": 0.623046875, + "0.6": 0.6904296875, + "0.7": 0.7464192708333334, + "0.8": 0.7970377604166666, + "0.9": 0.84228515625 + }, + "dedup_dcs_signed": { + "0.5": 0.5096158873056993, + "0.6": 0.5628642287920229, + "0.7": 0.6054426087260557, + "0.8": 0.6477385522254432, + "0.9": 0.6859118118882179 + }, + "matched_frac_signed": { + "0.5": 0.5524088541666666, + "0.6": 0.5882161458333334, + "0.7": 0.5862630208333334, + "0.8": 0.544921875, + "0.9": 0.4075520833333333 + }, + "matched_cos_abs": { + "0.5": 0.82192063331604, + "0.6": 0.8198916912078857, + "0.7": 0.8148212432861328, + "0.8": 0.816645085811615, + "0.9": 0.8187885880470276 + }, + "unique_frac_abs": { + "0.5": 0.6139322916666666, + "0.6": 0.6814778645833334, + "0.7": 0.7395833333333334, + "0.8": 0.791015625, + "0.9": 0.8395182291666666 + }, + "dedup_dcs_abs": { + "0.5": 0.5046036179798344, + "0.6": 0.5587380389139678, + "0.7": 0.6026282111803691, + "0.8": 0.6459790229564533, + "0.9": 0.6873879454991159 + }, + "matched_frac_abs": { + "0.5": 0.5478515625, + "0.6": 0.5869140625, + "0.7": 0.5862630208333334, + "0.8": 0.5439453125, + "0.9": 0.4075520833333333 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8456637263298035, + "dcs_abs": 0.8502656817436218, + "frac_sign_flipped": 0.031575520833333336, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8194010257720947, + "0.6": 0.8174140453338623, + "0.7": 0.8198081851005554, + "0.8": 0.8196272253990173, + "0.9": 0.8235411643981934 + }, + "unique_frac_signed": { + "0.5": 0.6223958333333334, + "0.6": 0.689453125, + "0.7": 0.748046875, + "0.8": 0.7970377604166666, + "0.9": 0.8406575520833334 + }, + "dedup_dcs_signed": { + "0.5": 0.5099917842696111, + "0.6": 0.563568667974323, + "0.7": 0.613254950963892, + "0.8": 0.6532738481085592, + "0.9": 0.6923160993028432 + }, + "matched_frac_signed": { + "0.5": 0.552734375, + "0.6": 0.5833333333333334, + "0.7": 0.5934244791666666, + "0.8": 0.5517578125, + "0.9": 0.4326171875 + }, + "matched_cos_abs": { + "0.5": 0.8246318697929382, + "0.6": 0.8222889304161072, + "0.7": 0.823699951171875, + "0.8": 0.8227087259292603, + "0.9": 0.8263576030731201 + }, + "unique_frac_abs": { + "0.5": 0.6136067708333334, + "0.6": 0.6805013020833334, + "0.7": 0.7415364583333334, + "0.8": 0.7919921875, + "0.9": 0.8361002604166666 + }, + "dedup_dcs_abs": { + "0.5": 0.5059996987498987, + "0.6": 0.5595686878368724, + "0.7": 0.6108035445213318, + "0.8": 0.6515788835240528, + "0.9": 0.6909178071267281 + }, + "matched_frac_abs": { + "0.5": 0.5475260416666666, + "0.6": 0.5826822916666666, + "0.7": 0.5934244791666666, + "0.8": 0.5517578125, + "0.9": 0.4296875 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.5894143581390381, + "babel_mu1_signed_std": 0.000972599159599066, + "n_dup_signed_per_threshold": { + "0.5": 1534.6666666666667, + "0.6": 1221.6666666666667, + "0.7": 962.6666666666666, + "0.8": 745.6666666666666, + "0.9": 564.3333333333334 + }, + "n_dup_signed_band": 873.5833333333334, + "n_dup_signed_band_std": 7.0371040445152815, + "n_dup_clusters_signed_per_threshold": { + "0.5": 378.0, + "0.6": 270.3333333333333, + "0.7": 187.33333333333334, + "0.8": 122.0, + "0.9": 76.66666666666667 + }, + "n_dup_clusters_signed_band": 164.08333333333331, + "n_dup_clusters_signed_band_std": 6.276610019216848, + "largest_dup_signed_per_threshold": { + "0.5": 164.66666666666666, + "0.6": 164.66666666666666, + "0.7": 129.0, + "0.8": 117.66666666666667, + "0.9": 73.33333333333333 + }, + "largest_dup_signed_band": 121.16666666666666, + "largest_dup_signed_band_std": 14.59951483212142, + "dcs_signed_mean": 0.8351107637087504, + "dcs_signed_std": 0.01251743631441493, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5050146831328878, + "0.6": 0.5581121506143568, + "0.7": 0.6035661642607819, + "0.8": 0.643524894422929, + "0.9": 0.6815789817483164 + }, + "dedup_dcs_signed_band": 0.621695547761596, + "dedup_dcs_signed_band_std": 0.011293201355414123, + "matched_cos_signed_per_threshold": { + "0.5": 0.8100135723749796, + "0.6": 0.8084909915924072, + "0.7": 0.8073296149571737, + "0.8": 0.8074495196342468, + "0.9": 0.8101921677589417 + }, + "matched_cos_signed_band": 0.8083655734856924, + "matched_cos_signed_band_std": 0.014864067072406676, + "unique_frac_signed_per_threshold": { + "0.5": 0.6234809027777778, + "0.6": 0.6903211805555555, + "0.7": 0.7476128472222223, + "0.8": 0.7969835069444443, + "0.9": 0.8412543402777778 + }, + "unique_frac_signed_band": 0.7690429687499999, + "unique_frac_signed_band_std": 0.000244140625, + "matched_frac_signed_per_threshold": { + "0.5": 0.550564236111111, + "0.6": 0.5819227430555557, + "0.7": 0.5810546875, + "0.8": 0.5330946180555555, + "0.9": 0.3938802083333333 + }, + "matched_frac_signed_band": 0.5224880642361112, + "matched_frac_signed_band_std": 0.0238084518478922, + "babel_mu1_abs_mean": 0.5964318712552389, + "babel_mu1_abs_std": 0.0007197430668216988, + "n_dup_abs_per_threshold": { + "0.5": 1572.3333333333333, + "0.6": 1263.3333333333333, + "0.7": 990.0, + "0.8": 764.6666666666666, + "0.9": 576.3333333333334 + }, + "n_dup_abs_band": 898.5833333333333, + "n_dup_abs_band_std": 5.513241635674364, + "n_dup_clusters_abs_per_threshold": { + "0.5": 388.3333333333333, + "0.6": 284.6666666666667, + "0.7": 194.0, + "0.8": 123.66666666666667, + "0.9": 78.0 + }, + "n_dup_clusters_abs_band": 170.08333333333334, + "n_dup_clusters_abs_band_std": 5.490522136676377, + "largest_dup_abs_per_threshold": { + "0.5": 186.0, + "0.6": 186.0, + "0.7": 178.33333333333334, + "0.8": 115.66666666666667, + "0.9": 70.33333333333333 + }, + "largest_dup_abs_band": 137.58333333333334, + "largest_dup_abs_band_std": 8.818777315100622, + "dcs_abs_mean": 0.839637815952301, + "dcs_abs_std": 0.012932466874856824, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5007632511034091, + "0.6": 0.5541028382172549, + "0.7": 0.601207784583999, + "0.8": 0.6422096409757311, + "0.9": 0.6821591577900108 + }, + "dedup_dcs_abs_band": 0.619919855391749, + "dedup_dcs_abs_band_std": 0.010688158828354681, + "matched_cos_abs_per_threshold": { + "0.5": 0.814823846022288, + "0.6": 0.8131654063860575, + "0.7": 0.8114749590555826, + "0.8": 0.8115412791570028, + "0.9": 0.8142487208048502 + }, + "matched_cos_abs_band": 0.8126075913508732, + "matched_cos_abs_band_std": 0.014273973985141597, + "unique_frac_abs_per_threshold": { + "0.5": 0.6145833333333334, + "0.6": 0.6814236111111112, + "0.7": 0.7408854166666666, + "0.8": 0.7913411458333334, + "0.9": 0.8377821180555555 + }, + "unique_frac_abs_band": 0.7628580729166666, + "unique_frac_abs_band_std": 0.00030720354961225385, + "matched_frac_abs_per_threshold": { + "0.5": 0.5453559027777777, + "0.6": 0.5811631944444443, + "0.7": 0.5815972222222222, + "0.8": 0.5317925347222222, + "0.9": 0.39203559027777773 + }, + "matched_frac_abs_band": 0.5216471354166666, + "matched_frac_abs_band_std": 0.02396491692796204, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 3072 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/config.json b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e29bc53bb30892fe2d225d5ad62f012886441434 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 32, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 32, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/model.safetensors b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b861996a69c314513ca276ad9b5dcd6a7df7d72f --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:571b80554e326186ad6cb1e6dd97ccd1077cd84889792108724c9a7422394c9e +size 18890032 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..06f8989dc94866076d7e98ef22333663a644ea54 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e982a82de9809f633de4fd56f6df1c53e313dbecefcbe678283ef3e1e3cf155 +size 25920 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..96227314759ef0055b546e776c0aa75436707ee8 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e99e11d35ca557f41c978a4a1b14e55733f696346a2478994990563efc4902ca +size 13607 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..31754bbf4121e2cd8ac2625e98f0cc15a9f707a9 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 11.807441845703124, + "l0_per_token_0.001": 11.807441845703124, + "l0_per_token_0.01": 11.807441845703124, + "l0_per_token_0.05": 11.807441845703124, + "l0_per_token_0.1": 11.807441845703124, + "l0_per_token_0.5": 11.8061119140625, + "l0_per_token_1.0": 11.75499580078125, + "l1_per_token": 44.71890905806031, + "r2_score": 0.8547333848294265, + "l2_loss": 0.09671429789500322 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..52ab11b4f323e38f175ebe8db931494909b31778 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d34e8f3c0a8c8925b126b8f42b13aa698e82bf8bfe7b15d0bf0681f15254844 +size 13518 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d678d3558d06d9563e7a5aed6967bfcfaa53d22f --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b18011d1cf121ff1f46329097f669e52640ac7391966ffd1f962ab5d2973ec8 +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6808967ab2c6ff1a5ca9bdb102393c66cbe3f7c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3684e2f12e6def98502a0b47ab2ee05cb3f3696e95f3b7029d41402343e66edc +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/config.json b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d39c416079930c68d7c93074e3b2b5646a514bb7 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 32, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 32, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/model.safetensors b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..95bf365c4211956f6cdbbee0803ab6e95ce932fc --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6566abdca75216b0030458c73cec7cb6320d013a3a498849a12b99112563f26a +size 18890032 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..93c35fd9e60b50670dc6a4d3df6aa2743d2846a5 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0866001df5e56e48203c3a60265a86fe4e842d969f6064d843cbae1b2eae9006 +size 25920 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..e14d94f220ea90370b93a5305adafd20761d60c6 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc8b8da5da5c6baf9efa1a657224294a61eefd2ec67219c3b1ba283519c3141b +size 13607 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..900307cdae545519e7c70736d42fa708052c44ad --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 12.081834521484375, + "l0_per_token_0.001": 12.081834521484375, + "l0_per_token_0.01": 12.081834521484375, + "l0_per_token_0.05": 12.081834521484375, + "l0_per_token_0.1": 12.081834521484375, + "l0_per_token_0.5": 12.081421044921875, + "l0_per_token_1.0": 12.020004345703125, + "l1_per_token": 45.32236212879355, + "r2_score": 0.8558574905656944, + "l2_loss": 0.09596590090843317 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..56eb2b7a0ced3d33ec7d1f24223c15ea73839711 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c103ed13fa45732ffbab83ab5754b6f6634918afb9930e4e70ffa839e9a6794f +size 13518 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd3c8662d57fb5e7bd3474b4caf6064251153592 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf4137fee1e012d6badfbebfe1095f132a1ca2f096b58771eaeeb5ca6f41bf26 +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fb28f8da2fadc95e15057edf1721af541b70c50 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:734882fe8dd56e1bd168cc69e1fba5945c534e2a08fd6aa60f2ba2ccf78dc223 +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/config.json b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbfe8b7905bce01e56935c8b4e6059bb883ce1f9 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 32, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 32, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/model.safetensors b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ad61fd48c1106d8228628eebdf6a0bad6ea9c88c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87740febb2cca29e4547f955f3b3634ef6aff8549ae123bde46b9f38df2948dc +size 18890032 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..07674f31c203a1db31e0af72712d7f5936fecfd6 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e898faf545aaa007d6f2827990fa9f79db066a023568887359f0c6dcdfc087d +size 25920 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa2f1e1a5535b1ed6f6c15347d75b853ef959db5 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b423798602411b4fe1c0b80de177fc7f6eefc23ebf3d4dcde13217d7202226f +size 13607 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..97a21616b33502a9bef42d8dcd5358c846804595 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 12.7585103515625, + "l0_per_token_0.001": 12.7585103515625, + "l0_per_token_0.01": 12.7585103515625, + "l0_per_token_0.05": 12.7585103515625, + "l0_per_token_0.1": 12.7585103515625, + "l0_per_token_0.5": 12.757329052734375, + "l0_per_token_1.0": 12.665068115234375, + "l1_per_token": 46.4482664841365, + "r2_score": 0.8583668927531972, + "l2_loss": 0.09429521373495188 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8433ebab4b42652cddf1b492eccabb1efe1145e1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1875ffbab06df3d9c8543d48322c3d0f4a3b7093645a3dcaa9bbd8de16751e36 +size 13518 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..611e31b1d8465a1131e82a9a33fd1007340fedce --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:508c3e59c421c46ed74596335024eadc0ae2190b553da82d4e7b874c1be08966 +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4da825740f41cea60ce172c919380a2512f1fa5 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:536eea5d44e26037d9617baa915937daca6081504969f53d5420f68ef0f7a57c +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/code_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x4/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..374f9395534178d36fea59df39a576d251849794 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 74.59344666666667, + "recon_r2": 0.9300341418178543, + "l1_per_token": 90.61562497306788 + }, + "seed1": { + "l0": 74.47944833333334, + "recon_r2": 0.9298792853295661, + "l1_per_token": 90.80396640408732 + }, + "seed2": { + "l0": 74.52445833333333, + "recon_r2": 0.9299858808227701, + "l1_per_token": 90.96650383977766 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.5095629276037345, + "mcc_abs": 0.50958084033195, + "act_iou": 0.18173930444543884, + "act_iou_n_pairs": 3072, + "input_r2": 0.9591212125781509, + "matched_mcc_signed": { + "0.6": 0.6751159937716986, + "0.7": 0.6877317280110115, + "0.8": 0.7060827731892402, + "0.9": 0.7322452661579198 + }, + "matched_mcc_abs": { + "0.6": 0.6736513309495497, + "0.7": 0.6857773057017217, + "0.8": 0.7057637926598147, + "0.9": 0.7352221765968884 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5077228670479319, + "mcc_abs": 0.5077579783694471, + "act_iou": 0.18160556076511405, + "act_iou_n_pairs": 3072, + "input_r2": 0.9595337169869665, + "matched_mcc_signed": { + "0.6": 0.6688639914284199, + "0.7": 0.6766782830651406, + "0.8": 0.6922198067221182, + "0.9": 0.7261391065480411 + }, + "matched_mcc_abs": { + "0.6": 0.6660090368323724, + "0.7": 0.6748041807175686, + "0.8": 0.691154152679693, + "0.9": 0.7240905033690975 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.5104842859544187, + "mcc_abs": 0.5105389665327583, + "act_iou": 0.18278688156784595, + "act_iou_n_pairs": 3072, + "input_r2": 0.9594291308051673, + "matched_mcc_signed": { + "0.6": 0.6709192438419083, + "0.7": 0.6676986098268255, + "0.8": 0.6865462476668585, + "0.9": 0.7166088647590287 + }, + "matched_mcc_abs": { + "0.6": 0.6690339678029686, + "0.7": 0.6700127117925008, + "0.8": 0.6862882359230246, + "0.9": 0.7178945609492063 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 74.53245111111111, + "l0_std": 0.05741792728014375, + "recon_r2_mean": 0.9299664359900635, + "recon_r2_std": 7.923830595411145e-05, + "l1_per_token_mean": 90.79536507231096, + "l1_per_token_std": 0.17559749986013412, + "input_r2_mean": 0.9593613534567615, + "input_r2_std": 0.0002144418302029267, + "mcc_signed_mean": 0.5092566935353617, + "mcc_signed_std": 0.0014059491714673918, + "mcc_abs_mean": 0.5092925950780517, + "mcc_abs_std": 0.0014127235348486732, + "matched_mcc_signed_per_threshold": { + "0.6": 0.6716330763473423, + "0.7": 0.6773695403009925, + "0.8": 0.6949496091927388, + "0.9": 0.7249977458216632 + }, + "matched_mcc_signed_band": 0.6922374929156841, + "matched_mcc_abs_per_threshold": { + "0.6": 0.6695647785282969, + "0.7": 0.6768647327372638, + "0.8": 0.6944020604208441, + "0.9": 0.7257357469717308 + }, + "matched_mcc_abs_band": 0.6916418296645339, + "act_iou_mean": 0.18204391559279962, + "act_iou_std": 0.0006468930924535495, + "act_iou_n_pairs": 3072.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/dict_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x4/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..527d0262e8dd62abb68a8ba572757a20309e4e75 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/dict_metrics_step_final.json @@ -0,0 +1,595 @@ +{ + "schema": 3, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1523, + "0.6": 1214, + "0.7": 970, + "0.8": 748, + "0.9": 559 + }, + "n_dup_clusters_signed": { + "0.5": 373, + "0.6": 268, + "0.7": 192, + "0.8": 124, + "0.9": 75 + }, + "n_dup_removed_signed": { + "0.5": 1150, + "0.6": 946, + "0.7": 778, + "0.8": 624, + "0.9": 484 + }, + "largest_dup_signed": { + "0.5": 199, + "0.6": 199, + "0.7": 101, + "0.8": 101, + "0.9": 61 + }, + "babel_mu1_signed": 0.5888772010803223, + "n_dup_abs": { + "0.5": 1565, + "0.6": 1261, + "0.7": 1001, + "0.8": 775, + "0.9": 563 + }, + "n_dup_clusters_abs": { + "0.5": 387, + "0.6": 288, + "0.7": 201, + "0.8": 130, + "0.9": 75 + }, + "n_dup_removed_abs": { + "0.5": 1178, + "0.6": 973, + "0.7": 800, + "0.8": 645, + "0.9": 488 + }, + "largest_dup_abs": { + "0.5": 219, + "0.6": 219, + "0.7": 196, + "0.8": 108, + "0.9": 68 + }, + "babel_mu1_abs": 0.5962309241294861, + "dup_excludes_dead": true + }, + "seed1": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1523, + "0.6": 1217, + "0.7": 943, + "0.8": 739, + "0.9": 569 + }, + "n_dup_clusters_signed": { + "0.5": 369, + "0.6": 265, + "0.7": 175, + "0.8": 115, + "0.9": 75 + }, + "n_dup_removed_signed": { + "0.5": 1154, + "0.6": 952, + "0.7": 768, + "0.8": 624, + "0.9": 494 + }, + "largest_dup_signed": { + "0.5": 131, + "0.6": 131, + "0.7": 122, + "0.8": 98, + "0.9": 90 + }, + "babel_mu1_signed": 0.5888288021087646, + "n_dup_abs": { + "0.5": 1562, + "0.6": 1257, + "0.7": 971, + "0.8": 757, + "0.9": 585 + }, + "n_dup_clusters_abs": { + "0.5": 382, + "0.6": 278, + "0.7": 183, + "0.8": 118, + "0.9": 76 + }, + "n_dup_removed_abs": { + "0.5": 1180, + "0.6": 979, + "0.7": 788, + "0.8": 639, + "0.9": 509 + }, + "largest_dup_abs": { + "0.5": 165, + "0.6": 165, + "0.7": 165, + "0.8": 142, + "0.9": 60 + }, + "babel_mu1_abs": 0.595833957195282, + "dup_excludes_dead": true + }, + "seed2": { + "d_sae": 3072, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 1558, + "0.6": 1234, + "0.7": 975, + "0.8": 750, + "0.9": 565 + }, + "n_dup_clusters_signed": { + "0.5": 392, + "0.6": 278, + "0.7": 195, + "0.8": 127, + "0.9": 80 + }, + "n_dup_removed_signed": { + "0.5": 1166, + "0.6": 956, + "0.7": 780, + "0.8": 623, + "0.9": 485 + }, + "largest_dup_signed": { + "0.5": 164, + "0.6": 164, + "0.7": 164, + "0.8": 154, + "0.9": 69 + }, + "babel_mu1_signed": 0.5905370712280273, + "n_dup_abs": { + "0.5": 1590, + "0.6": 1272, + "0.7": 998, + "0.8": 762, + "0.9": 581 + }, + "n_dup_clusters_abs": { + "0.5": 396, + "0.6": 288, + "0.7": 198, + "0.8": 123, + "0.9": 83 + }, + "n_dup_removed_abs": { + "0.5": 1194, + "0.6": 984, + "0.7": 800, + "0.8": 639, + "0.9": 498 + }, + "largest_dup_abs": { + "0.5": 174, + "0.6": 174, + "0.7": 174, + "0.8": 97, + "0.9": 83 + }, + "babel_mu1_abs": 0.5972307324409485, + "dup_excludes_dead": true + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8212807774543762, + "dcs_abs": 0.8252390027046204, + "frac_sign_flipped": 0.037434895833333336, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.7926982045173645, + "0.6": 0.7928213477134705, + "0.7": 0.7910512685775757, + "0.8": 0.7900389432907104, + "0.9": 0.7926890254020691 + }, + "unique_frac_signed": { + "0.5": 0.625, + "0.6": 0.6910807291666666, + "0.7": 0.7483723958333334, + "0.8": 0.796875, + "0.9": 0.8408203125 + }, + "dedup_dcs_signed": { + "0.5": 0.4954363778233528, + "0.6": 0.5479035550767245, + "0.7": 0.592000933092398, + "0.8": 0.6295622829347849, + "0.9": 0.6665090340538882 + }, + "matched_frac_signed": { + "0.5": 0.5465494791666666, + "0.6": 0.57421875, + "0.7": 0.5634765625, + "0.8": 0.5026041666666666, + "0.9": 0.3414713541666667 + }, + "matched_cos_abs": { + "0.5": 0.7979190349578857, + "0.6": 0.7973155975341797, + "0.7": 0.7959036827087402, + "0.8": 0.7952700257301331, + "0.9": 0.7975999712944031 + }, + "unique_frac_abs": { + "0.5": 0.6162109375, + "0.6": 0.6822916666666666, + "0.7": 0.7415364583333334, + "0.8": 0.791015625, + "0.9": 0.8377278645833334 + }, + "dedup_dcs_abs": { + "0.5": 0.49168643658049405, + "0.6": 0.5440017879009247, + "0.7": 0.5901915980502963, + "0.8": 0.6290710164466873, + "0.9": 0.6681717207441883 + }, + "matched_frac_abs": { + "0.5": 0.5406901041666666, + "0.6": 0.5738932291666666, + "0.7": 0.5651041666666666, + "0.8": 0.4996744791666667, + "0.9": 0.3388671875 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8383877873420715, + "dcs_abs": 0.8434087634086609, + "frac_sign_flipped": 0.030598958333333332, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8179414868354797, + "0.6": 0.8152375817298889, + "0.7": 0.8111293911933899, + "0.8": 0.8126823902130127, + "0.9": 0.8143463134765625 + }, + "unique_frac_signed": { + "0.5": 0.623046875, + "0.6": 0.6904296875, + "0.7": 0.7464192708333334, + "0.8": 0.7970377604166666, + "0.9": 0.84228515625 + }, + "dedup_dcs_signed": { + "0.5": 0.5096158873056993, + "0.6": 0.5628642287920229, + "0.7": 0.6054426087260557, + "0.8": 0.6477385522254432, + "0.9": 0.6859118118882179 + }, + "matched_frac_signed": { + "0.5": 0.5524088541666666, + "0.6": 0.5882161458333334, + "0.7": 0.5862630208333334, + "0.8": 0.544921875, + "0.9": 0.4075520833333333 + }, + "matched_cos_abs": { + "0.5": 0.82192063331604, + "0.6": 0.8198916912078857, + "0.7": 0.8148212432861328, + "0.8": 0.816645085811615, + "0.9": 0.8187885880470276 + }, + "unique_frac_abs": { + "0.5": 0.6139322916666666, + "0.6": 0.6814778645833334, + "0.7": 0.7395833333333334, + "0.8": 0.791015625, + "0.9": 0.8395182291666666 + }, + "dedup_dcs_abs": { + "0.5": 0.5046036179798344, + "0.6": 0.5587380389139678, + "0.7": 0.6026282111803691, + "0.8": 0.6459790229564533, + "0.9": 0.6873879454991159 + }, + "matched_frac_abs": { + "0.5": 0.5478515625, + "0.6": 0.5869140625, + "0.7": 0.5862630208333334, + "0.8": 0.5439453125, + "0.9": 0.4075520833333333 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8456637263298035, + "dcs_abs": 0.8502656817436218, + "frac_sign_flipped": 0.031575520833333336, + "alive_source": "stats", + "frac_denominator": "d_sae", + "n_alive_a": 3072, + "n_alive_b": 3072, + "matched_cos_signed": { + "0.5": 0.8194010257720947, + "0.6": 0.8174140453338623, + "0.7": 0.8198081851005554, + "0.8": 0.8196272253990173, + "0.9": 0.8235411643981934 + }, + "unique_frac_signed": { + "0.5": 0.6223958333333334, + "0.6": 0.689453125, + "0.7": 0.748046875, + "0.8": 0.7970377604166666, + "0.9": 0.8406575520833334 + }, + "dedup_dcs_signed": { + "0.5": 0.5099917842696111, + "0.6": 0.563568667974323, + "0.7": 0.613254950963892, + "0.8": 0.6532738481085592, + "0.9": 0.6923160993028432 + }, + "matched_frac_signed": { + "0.5": 0.552734375, + "0.6": 0.5833333333333334, + "0.7": 0.5934244791666666, + "0.8": 0.5517578125, + "0.9": 0.4326171875 + }, + "matched_cos_abs": { + "0.5": 0.8246318697929382, + "0.6": 0.8222889304161072, + "0.7": 0.823699951171875, + "0.8": 0.8227087259292603, + "0.9": 0.8263576030731201 + }, + "unique_frac_abs": { + "0.5": 0.6136067708333334, + "0.6": 0.6805013020833334, + "0.7": 0.7415364583333334, + "0.8": 0.7919921875, + "0.9": 0.8361002604166666 + }, + "dedup_dcs_abs": { + "0.5": 0.5059996987498987, + "0.6": 0.5595686878368724, + "0.7": 0.6108035445213318, + "0.8": 0.6515788835240528, + "0.9": 0.6909178071267281 + }, + "matched_frac_abs": { + "0.5": 0.5475260416666666, + "0.6": 0.5826822916666666, + "0.7": 0.5934244791666666, + "0.8": 0.5517578125, + "0.9": 0.4296875 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.5894143581390381, + "babel_mu1_signed_std": 0.000972599159599066, + "n_dup_signed_per_threshold": { + "0.5": 1534.6666666666667, + "0.6": 1221.6666666666667, + "0.7": 962.6666666666666, + "0.8": 745.6666666666666, + "0.9": 564.3333333333334 + }, + "n_dup_signed_band": 873.5833333333334, + "n_dup_signed_band_std": 7.0371040445152815, + "n_dup_clusters_signed_per_threshold": { + "0.5": 378.0, + "0.6": 270.3333333333333, + "0.7": 187.33333333333334, + "0.8": 122.0, + "0.9": 76.66666666666667 + }, + "n_dup_clusters_signed_band": 164.08333333333331, + "n_dup_clusters_signed_band_std": 6.276610019216848, + "largest_dup_signed_per_threshold": { + "0.5": 164.66666666666666, + "0.6": 164.66666666666666, + "0.7": 129.0, + "0.8": 117.66666666666667, + "0.9": 73.33333333333333 + }, + "largest_dup_signed_band": 121.16666666666666, + "largest_dup_signed_band_std": 14.59951483212142, + "dcs_signed_mean": 0.8351107637087504, + "dcs_signed_std": 0.01251743631441493, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.5050146831328878, + "0.6": 0.5581121506143568, + "0.7": 0.6035661642607819, + "0.8": 0.643524894422929, + "0.9": 0.6815789817483164 + }, + "dedup_dcs_signed_band": 0.621695547761596, + "dedup_dcs_signed_band_std": 0.011293201355414123, + "matched_cos_signed_per_threshold": { + "0.5": 0.8100135723749796, + "0.6": 0.8084909915924072, + "0.7": 0.8073296149571737, + "0.8": 0.8074495196342468, + "0.9": 0.8101921677589417 + }, + "matched_cos_signed_band": 0.8083655734856924, + "matched_cos_signed_band_std": 0.014864067072406676, + "unique_frac_signed_per_threshold": { + "0.5": 0.6234809027777778, + "0.6": 0.6903211805555555, + "0.7": 0.7476128472222223, + "0.8": 0.7969835069444443, + "0.9": 0.8412543402777778 + }, + "unique_frac_signed_band": 0.7690429687499999, + "unique_frac_signed_band_std": 0.000244140625, + "matched_frac_signed_per_threshold": { + "0.5": 0.550564236111111, + "0.6": 0.5819227430555557, + "0.7": 0.5810546875, + "0.8": 0.5330946180555555, + "0.9": 0.3938802083333333 + }, + "matched_frac_signed_band": 0.5224880642361112, + "matched_frac_signed_band_std": 0.0238084518478922, + "babel_mu1_abs_mean": 0.5964318712552389, + "babel_mu1_abs_std": 0.0007197430668216988, + "n_dup_abs_per_threshold": { + "0.5": 1572.3333333333333, + "0.6": 1263.3333333333333, + "0.7": 990.0, + "0.8": 764.6666666666666, + "0.9": 576.3333333333334 + }, + "n_dup_abs_band": 898.5833333333333, + "n_dup_abs_band_std": 5.513241635674364, + "n_dup_clusters_abs_per_threshold": { + "0.5": 388.3333333333333, + "0.6": 284.6666666666667, + "0.7": 194.0, + "0.8": 123.66666666666667, + "0.9": 78.0 + }, + "n_dup_clusters_abs_band": 170.08333333333334, + "n_dup_clusters_abs_band_std": 5.490522136676377, + "largest_dup_abs_per_threshold": { + "0.5": 186.0, + "0.6": 186.0, + "0.7": 178.33333333333334, + "0.8": 115.66666666666667, + "0.9": 70.33333333333333 + }, + "largest_dup_abs_band": 137.58333333333334, + "largest_dup_abs_band_std": 8.818777315100622, + "dcs_abs_mean": 0.839637815952301, + "dcs_abs_std": 0.012932466874856824, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.5007632511034091, + "0.6": 0.5541028382172549, + "0.7": 0.601207784583999, + "0.8": 0.6422096409757311, + "0.9": 0.6821591577900108 + }, + "dedup_dcs_abs_band": 0.619919855391749, + "dedup_dcs_abs_band_std": 0.010688158828354681, + "matched_cos_abs_per_threshold": { + "0.5": 0.814823846022288, + "0.6": 0.8131654063860575, + "0.7": 0.8114749590555826, + "0.8": 0.8115412791570028, + "0.9": 0.8142487208048502 + }, + "matched_cos_abs_band": 0.8126075913508732, + "matched_cos_abs_band_std": 0.014273973985141597, + "unique_frac_abs_per_threshold": { + "0.5": 0.6145833333333334, + "0.6": 0.6814236111111112, + "0.7": 0.7408854166666666, + "0.8": 0.7913411458333334, + "0.9": 0.8377821180555555 + }, + "unique_frac_abs_band": 0.7628580729166666, + "unique_frac_abs_band_std": 0.00030720354961225385, + "matched_frac_abs_per_threshold": { + "0.5": 0.5453559027777777, + "0.6": 0.5811631944444443, + "0.7": 0.5815972222222222, + "0.8": 0.5317925347222222, + "0.9": 0.39203559027777773 + }, + "matched_frac_abs_band": 0.5216471354166666, + "matched_frac_abs_band_std": 0.02396491692796204, + "alive_source": "stats", + "frac_denominator": "d_sae", + "d_sae": 3072 + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/config.json b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e29bc53bb30892fe2d225d5ad62f012886441434 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 32, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 32, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/model.safetensors b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b861996a69c314513ca276ad9b5dcd6a7df7d72f --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:571b80554e326186ad6cb1e6dd97ccd1077cd84889792108724c9a7422394c9e +size 18890032 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..06f8989dc94866076d7e98ef22333663a644ea54 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e982a82de9809f633de4fd56f6df1c53e313dbecefcbe678283ef3e1e3cf155 +size 25920 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..96227314759ef0055b546e776c0aa75436707ee8 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e99e11d35ca557f41c978a4a1b14e55733f696346a2478994990563efc4902ca +size 13607 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..31754bbf4121e2cd8ac2625e98f0cc15a9f707a9 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 11.807441845703124, + "l0_per_token_0.001": 11.807441845703124, + "l0_per_token_0.01": 11.807441845703124, + "l0_per_token_0.05": 11.807441845703124, + "l0_per_token_0.1": 11.807441845703124, + "l0_per_token_0.5": 11.8061119140625, + "l0_per_token_1.0": 11.75499580078125, + "l1_per_token": 44.71890905806031, + "r2_score": 0.8547333848294265, + "l2_loss": 0.09671429789500322 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..52ab11b4f323e38f175ebe8db931494909b31778 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d34e8f3c0a8c8925b126b8f42b13aa698e82bf8bfe7b15d0bf0681f15254844 +size 13518 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d678d3558d06d9563e7a5aed6967bfcfaa53d22f --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b18011d1cf121ff1f46329097f669e52640ac7391966ffd1f962ab5d2973ec8 +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6808967ab2c6ff1a5ca9bdb102393c66cbe3f7c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3684e2f12e6def98502a0b47ab2ee05cb3f3696e95f3b7029d41402343e66edc +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/config.json b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d39c416079930c68d7c93074e3b2b5646a514bb7 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 32, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 32, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/model.safetensors b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..95bf365c4211956f6cdbbee0803ab6e95ce932fc --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6566abdca75216b0030458c73cec7cb6320d013a3a498849a12b99112563f26a +size 18890032 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..93c35fd9e60b50670dc6a4d3df6aa2743d2846a5 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0866001df5e56e48203c3a60265a86fe4e842d969f6064d843cbae1b2eae9006 +size 25920 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..e14d94f220ea90370b93a5305adafd20761d60c6 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc8b8da5da5c6baf9efa1a657224294a61eefd2ec67219c3b1ba283519c3141b +size 13607 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..900307cdae545519e7c70736d42fa708052c44ad --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 12.081834521484375, + "l0_per_token_0.001": 12.081834521484375, + "l0_per_token_0.01": 12.081834521484375, + "l0_per_token_0.05": 12.081834521484375, + "l0_per_token_0.1": 12.081834521484375, + "l0_per_token_0.5": 12.081421044921875, + "l0_per_token_1.0": 12.020004345703125, + "l1_per_token": 45.32236212879355, + "r2_score": 0.8558574905656944, + "l2_loss": 0.09596590090843317 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..56eb2b7a0ced3d33ec7d1f24223c15ea73839711 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c103ed13fa45732ffbab83ab5754b6f6634918afb9930e4e70ffa839e9a6794f +size 13518 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd3c8662d57fb5e7bd3474b4caf6064251153592 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf4137fee1e012d6badfbebfe1095f132a1ca2f096b58771eaeeb5ca6f41bf26 +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fb28f8da2fadc95e15057edf1721af541b70c50 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:734882fe8dd56e1bd168cc69e1fba5945c534e2a08fd6aa60f2ba2ccf78dc223 +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/config.json b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbfe8b7905bce01e56935c8b4e6059bb883ce1f9 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 4, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 32, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 32, + "residual_thr": 0.05, + "max_iters": 307, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/model.safetensors b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ad61fd48c1106d8228628eebdf6a0bad6ea9c88c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87740febb2cca29e4547f955f3b3634ef6aff8549ae123bde46b9f38df2948dc +size 18890032 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..07674f31c203a1db31e0af72712d7f5936fecfd6 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e898faf545aaa007d6f2827990fa9f79db066a023568887359f0c6dcdfc087d +size 25920 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa2f1e1a5535b1ed6f6c15347d75b853ef959db5 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b423798602411b4fe1c0b80de177fc7f6eefc23ebf3d4dcde13217d7202226f +size 13607 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..97a21616b33502a9bef42d8dcd5358c846804595 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 12.7585103515625, + "l0_per_token_0.001": 12.7585103515625, + "l0_per_token_0.01": 12.7585103515625, + "l0_per_token_0.05": 12.7585103515625, + "l0_per_token_0.1": 12.7585103515625, + "l0_per_token_0.5": 12.757329052734375, + "l0_per_token_1.0": 12.665068115234375, + "l1_per_token": 46.4482664841365, + "r2_score": 0.8583668927531972, + "l2_loss": 0.09429521373495188 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8433ebab4b42652cddf1b492eccabb1efe1145e1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1875ffbab06df3d9c8543d48322c3d0f4a3b7093645a3dcaa9bbd8de16751e36 +size 13518 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..611e31b1d8465a1131e82a9a33fd1007340fedce --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:508c3e59c421c46ed74596335024eadc0ae2190b553da82d4e7b874c1be08966 +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..642c940f0af51c624c56af1eb48cf5cc4c88531d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 3072, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4da825740f41cea60ce172c919380a2512f1fa5 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:536eea5d44e26037d9617baa915937daca6081504969f53d5420f68ef0f7a57c +size 13523 diff --git a/lm/pythia-160m_layer8/mp/x8/l0_128/code_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x8/l0_128/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..70c42da87f9c964e33e4cc2b7bb7c83ac666c396 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_128/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 141.67612166666666, + "recon_r2": 0.9515022750498221, + "l1_per_token": 108.86267195249019 + }, + "seed1": { + "l0": 145.15866166666666, + "recon_r2": 0.9519063168093808, + "l1_per_token": 109.98228528452759 + }, + "seed2": { + "l0": 142.272925, + "recon_r2": 0.9519674096786727, + "l1_per_token": 109.15221582647841 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.46584384411656554, + "mcc_abs": 0.46587251649930783, + "act_iou": 0.12373119931316241, + "act_iou_n_pairs": 6144, + "input_r2": 0.9699977832547844, + "matched_mcc_signed": { + "0.6": 0.6161847228881695, + "0.7": 0.627672492947477, + "0.8": 0.6356583085944716, + "0.9": 0.6531931157131006 + }, + "matched_mcc_abs": { + "0.6": 0.6162552811127698, + "0.7": 0.626853819702229, + "0.8": 0.6365088452272449, + "0.9": 0.6530337102218212 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.4825729185943131, + "mcc_abs": 0.482592264418467, + "act_iou": 0.12935204818082505, + "act_iou_n_pairs": 6144, + "input_r2": 0.9704180507832192, + "matched_mcc_signed": { + "0.6": 0.6173570075995674, + "0.7": 0.6256302493184068, + "0.8": 0.6321348052456569, + "0.9": 0.6463717430998857 + }, + "matched_mcc_abs": { + "0.6": 0.6167434612859812, + "0.7": 0.6247887407680344, + "0.8": 0.6326111799140431, + "0.9": 0.6482760781655199 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.4694000710946156, + "mcc_abs": 0.4694255791807123, + "act_iou": 0.12436839275674312, + "act_iou_n_pairs": 6144, + "input_r2": 0.970172886562099, + "matched_mcc_signed": { + "0.6": 0.6147991360722004, + "0.7": 0.6234505420149811, + "0.8": 0.6328901849035238, + "0.9": 0.6469418418792791 + }, + "matched_mcc_abs": { + "0.6": 0.613698453513571, + "0.7": 0.6236734761961792, + "0.8": 0.6341238044623545, + "0.9": 0.647859825894199 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 143.03590277777778, + "l0_std": 1.862423831982678, + "recon_r2_mean": 0.9517920005126251, + "recon_r2_std": 0.0002527621758035025, + "l1_per_token_mean": 109.33239102116539, + "l1_per_token_std": 0.5811461338633718, + "input_r2_mean": 0.9701962402000341, + "input_r2_std": 0.0002111048155959148, + "mcc_signed_mean": 0.47260561126849804, + "mcc_signed_std": 0.008813177581059352, + "mcc_abs_mean": 0.47263012003282906, + "mcc_abs_std": 0.00880847910678798, + "matched_mcc_signed_per_threshold": { + "0.6": 0.6161136221866457, + "0.7": 0.6255844280936217, + "0.8": 0.6335610995812174, + "0.9": 0.6488355668974218 + }, + "matched_mcc_signed_band": 0.6310236791897267, + "matched_mcc_abs_per_threshold": { + "0.6": 0.615565731970774, + "0.7": 0.6251053455554808, + "0.8": 0.6344146098678809, + "0.9": 0.6497232047605134 + }, + "matched_mcc_abs_band": 0.6312022230386622, + "act_iou_mean": 0.12581721341691018, + "act_iou_std": 0.00307779084365252, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_128/dict_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x8/l0_128/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..0dec86e75d9da7257bdb41d7149ebab1821a300d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_128/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4142, + "0.6": 3246, + "0.7": 2475, + "0.8": 1880, + "0.9": 1532 + }, + "n_dup_clusters_signed": { + "0.5": 1049, + "0.6": 733, + "0.7": 450, + "0.8": 222, + "0.9": 121 + }, + "largest_dup_signed": { + "0.5": 522, + "0.6": 522, + "0.7": 494, + "0.8": 274, + "0.9": 195 + }, + "babel_mu1_signed": 0.6800978779792786, + "n_dup_abs": { + "0.5": 4160, + "0.6": 3268, + "0.7": 2495, + "0.8": 1895, + "0.9": 1551 + }, + "n_dup_clusters_abs": { + "0.5": 1053, + "0.6": 734, + "0.7": 451, + "0.8": 222, + "0.9": 126 + }, + "largest_dup_abs": { + "0.5": 604, + "0.6": 604, + "0.7": 361, + "0.8": 337, + "0.9": 195 + }, + "babel_mu1_abs": 0.6818435788154602 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4124, + "0.6": 3244, + "0.7": 2498, + "0.8": 1908, + "0.9": 1534 + }, + "n_dup_clusters_signed": { + "0.5": 1026, + "0.6": 727, + "0.7": 458, + "0.8": 235, + "0.9": 116 + }, + "largest_dup_signed": { + "0.5": 490, + "0.6": 490, + "0.7": 490, + "0.8": 352, + "0.9": 125 + }, + "babel_mu1_signed": 0.679654598236084, + "n_dup_abs": { + "0.5": 4137, + "0.6": 3270, + "0.7": 2517, + "0.8": 1921, + "0.9": 1546 + }, + "n_dup_clusters_abs": { + "0.5": 1024, + "0.6": 730, + "0.7": 459, + "0.8": 233, + "0.9": 118 + }, + "largest_dup_abs": { + "0.5": 524, + "0.6": 503, + "0.7": 499, + "0.8": 302, + "0.9": 122 + }, + "babel_mu1_abs": 0.6812613606452942 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4150, + "0.6": 3241, + "0.7": 2441, + "0.8": 1818, + "0.9": 1462 + }, + "n_dup_clusters_signed": { + "0.5": 1084, + "0.6": 771, + "0.7": 473, + "0.8": 231, + "0.9": 119 + }, + "largest_dup_signed": { + "0.5": 582, + "0.6": 399, + "0.7": 322, + "0.8": 322, + "0.9": 170 + }, + "babel_mu1_signed": 0.67706698179245, + "n_dup_abs": { + "0.5": 4169, + "0.6": 3258, + "0.7": 2458, + "0.8": 1831, + "0.9": 1475 + }, + "n_dup_clusters_abs": { + "0.5": 1087, + "0.6": 768, + "0.7": 473, + "0.8": 232, + "0.9": 123 + }, + "largest_dup_abs": { + "0.5": 584, + "0.6": 584, + "0.7": 334, + "0.8": 237, + "0.9": 170 + }, + "babel_mu1_abs": 0.6785966753959656 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8674505352973938, + "dcs_abs": 0.8728463053703308, + "frac_sign_flipped": 0.02392578125, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8343208432197571, + "0.6": 0.8349205255508423, + "0.7": 0.8357803821563721, + "0.8": 0.8370693325996399, + "0.9": 0.8390234708786011 + }, + "unique_frac_signed": { + "0.5": 0.4961751302083333, + "0.6": 0.5906575520833334, + "0.7": 0.669189453125, + "0.8": 0.7289225260416666, + "0.9": 0.769775390625 + }, + "dedup_dcs_signed": { + "0.5": 0.4139692530200894, + "0.6": 0.4931521138059907, + "0.7": 0.5592954168678261, + "0.8": 0.6101586923905415, + "0.9": 0.6458596200391185 + }, + "matched_frac_signed": { + "0.5": 0.4453125, + "0.6": 0.5128580729166666, + "0.7": 0.54443359375, + "0.8": 0.5310872395833334, + "0.9": 0.44091796875 + }, + "matched_cos_abs": { + "0.5": 0.8370665907859802, + "0.6": 0.8378727436065674, + "0.7": 0.8381679654121399, + "0.8": 0.8394980430603027, + "0.9": 0.8412030935287476 + }, + "unique_frac_abs": { + "0.5": 0.4938151041666667, + "0.6": 0.5870768229166666, + "0.7": 0.6661783854166666, + "0.8": 0.7264811197916666, + "0.9": 0.767822265625 + }, + "dedup_dcs_abs": { + "0.5": 0.4133561257234154, + "0.6": 0.49189566832501436, + "0.7": 0.5583693819062319, + "0.8": 0.6098794783853615, + "0.9": 0.6458944651240017 + }, + "matched_frac_abs": { + "0.5": 0.4446614583333333, + "0.6": 0.5120442708333334, + "0.7": 0.5458984375, + "0.8": 0.5299479166666666, + "0.9": 0.43994140625 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8815560340881348, + "dcs_abs": 0.8838018774986267, + "frac_sign_flipped": 0.019694010416666668, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8520606756210327, + "0.6": 0.8557045459747314, + "0.7": 0.8584762215614319, + "0.8": 0.860958456993103, + "0.9": 0.862807035446167 + }, + "unique_frac_signed": { + "0.5": 0.498779296875, + "0.6": 0.594482421875, + "0.7": 0.675048828125, + "0.8": 0.7359212239583334, + "0.9": 0.77587890625 + }, + "dedup_dcs_signed": { + "0.5": 0.42499022468109615, + "0.6": 0.5087013109005056, + "0.7": 0.5795133673382225, + "0.8": 0.6335976014476425, + "0.9": 0.669433778966777 + }, + "matched_frac_signed": { + "0.5": 0.4475911458333333, + "0.6": 0.51953125, + "0.7": 0.5626627604166666, + "0.8": 0.5559895833333334, + "0.9": 0.4890950520833333 + }, + "matched_cos_abs": { + "0.5": 0.8549606800079346, + "0.6": 0.8587138652801514, + "0.7": 0.8611496090888977, + "0.8": 0.8635424375534058, + "0.9": 0.8655458688735962 + }, + "unique_frac_abs": { + "0.5": 0.496337890625, + "0.6": 0.5911458333333334, + "0.7": 0.672119140625, + "0.8": 0.7337239583333334, + "0.9": 0.7740071614583334 + }, + "dedup_dcs_abs": { + "0.5": 0.42434938048245385, + "0.6": 0.5076251234859228, + "0.7": 0.5787951352103846, + "0.8": 0.6336017754705002, + "0.9": 0.669938701078839 + }, + "matched_frac_abs": { + "0.5": 0.4462890625, + "0.6": 0.5196940104166666, + "0.7": 0.5633138020833334, + "0.8": 0.5556640625, + "0.9": 0.4871419270833333 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8638496398925781, + "dcs_abs": 0.8671876788139343, + "frac_sign_flipped": 0.025227864583333332, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8361403346061707, + "0.6": 0.8372998833656311, + "0.7": 0.8382330536842346, + "0.8": 0.8397597670555115, + "0.9": 0.8406630158424377 + }, + "unique_frac_signed": { + "0.5": 0.4983723958333333, + "0.6": 0.5941569010416666, + "0.7": 0.673828125, + "0.8": 0.7347005208333334, + "0.9": 0.7753092447916666 + }, + "dedup_dcs_signed": { + "0.5": 0.4167092618105622, + "0.6": 0.4974875039430723, + "0.7": 0.5648250068770722, + "0.8": 0.616971938230563, + "0.9": 0.6517738079370853 + }, + "matched_frac_signed": { + "0.5": 0.4431966145833333, + "0.6": 0.5143229166666666, + "0.7": 0.54931640625, + "0.8": 0.5323893229166666, + "0.9": 0.4298502604166667 + }, + "matched_cos_abs": { + "0.5": 0.8397873640060425, + "0.6": 0.84168541431427, + "0.7": 0.8415432572364807, + "0.8": 0.8431254029273987, + "0.9": 0.8439717292785645 + }, + "unique_frac_abs": { + "0.5": 0.495849609375, + "0.6": 0.5906575520833334, + "0.7": 0.6709798177083334, + "0.8": 0.7325032552083334, + "0.9": 0.7737630208333334 + }, + "dedup_dcs_abs": { + "0.5": 0.4164082364004571, + "0.6": 0.497147846443113, + "0.7": 0.5646585413342109, + "0.8": 0.6175921021931572, + "0.9": 0.6530341147445142 + }, + "matched_frac_abs": { + "0.5": 0.4431966145833333, + "0.6": 0.5146484375, + "0.7": 0.5515950520833334, + "0.8": 0.53271484375, + "0.9": 0.4290364583333333 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.6789398193359375, + "babel_mu1_signed_std": 0.001636998650245021, + "n_dup_signed_per_threshold": { + "0.5": 4138.666666666667, + "0.6": 3243.6666666666665, + "0.7": 2471.3333333333335, + "0.8": 1868.6666666666667, + "0.9": 1509.3333333333333 + }, + "n_dup_signed_band": 2273.25, + "n_dup_signed_band_std": 29.069958720300928, + "n_dup_clusters_signed_per_threshold": { + "0.5": 1053.0, + "0.6": 743.6666666666666, + "0.7": 460.3333333333333, + "0.8": 229.33333333333334, + "0.9": 118.66666666666667 + }, + "n_dup_clusters_signed_band": 388.0, + "n_dup_clusters_signed_band_std": 9.17877987534291, + "largest_dup_signed_per_threshold": { + "0.5": 531.3333333333334, + "0.6": 470.3333333333333, + "0.7": 435.3333333333333, + "0.8": 316.0, + "0.9": 163.33333333333334 + }, + "largest_dup_signed_band": 346.24999999999994, + "largest_dup_signed_band_std": 37.403208418530085, + "dcs_signed_mean": 0.8709520697593689, + "dcs_signed_std": 0.009358133176195448, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.41855624650391593, + "0.6": 0.49978030954985614, + "0.7": 0.5678779303610403, + "0.8": 0.6202427440229157, + "0.9": 0.6556890689809936 + }, + "dedup_dcs_signed_band": 0.5858975132287014, + "dedup_dcs_signed_band_std": 0.010697328677874826, + "matched_cos_signed_per_threshold": { + "0.5": 0.8408406178156534, + "0.6": 0.8426416516304016, + "0.7": 0.8441632191340128, + "0.8": 0.8459291855494181, + "0.9": 0.847497840722402 + }, + "matched_cos_signed_band": 0.8450579742590586, + "matched_cos_signed_band_std": 0.012547899147488203, + "unique_frac_signed_per_threshold": { + "0.5": 0.49777560763888884, + "0.6": 0.5930989583333334, + "0.7": 0.6726888020833334, + "0.8": 0.7331814236111112, + "0.9": 0.7736545138888888 + }, + "unique_frac_signed_band": 0.6931559244791667, + "unique_frac_signed_band_std": 0.0030765459213382804, + "matched_frac_signed_per_threshold": { + "0.5": 0.44536675347222215, + "0.6": 0.5155707465277777, + "0.7": 0.5521375868055555, + "0.8": 0.5398220486111112, + "0.9": 0.4532877604166667 + }, + "matched_frac_signed_band": 0.5152045355902777, + "matched_frac_signed_band_std": 0.0143954626508884, + "babel_mu1_abs_mean": 0.68056720495224, + "babel_mu1_abs_std": 0.0017311801028579161, + "n_dup_abs_per_threshold": { + "0.5": 4155.333333333333, + "0.6": 3265.3333333333335, + "0.7": 2490.0, + "0.8": 1882.3333333333333, + "0.9": 1524.0 + }, + "n_dup_abs_band": 2290.416666666667, + "n_dup_abs_band_std": 30.757451671641025, + "n_dup_clusters_abs_per_threshold": { + "0.5": 1054.6666666666667, + "0.6": 744.0, + "0.7": 461.0, + "0.8": 229.0, + "0.9": 122.33333333333333 + }, + "n_dup_clusters_abs_band": 389.0833333333333, + "n_dup_clusters_abs_band_std": 8.632545009053434, + "largest_dup_abs_per_threshold": { + "0.5": 570.6666666666666, + "0.6": 563.6666666666666, + "0.7": 398.0, + "0.8": 292.0, + "0.9": 162.33333333333334 + }, + "largest_dup_abs_band": 353.99999999999994, + "largest_dup_abs_band_std": 21.608736659046034, + "dcs_abs_mean": 0.8746119538942972, + "dcs_abs_std": 0.008446658245498468, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.41803791420210884, + "0.6": 0.4988895460846834, + "0.7": 0.5672743528169425, + "0.8": 0.620357785349673, + "0.9": 0.6562890936491184 + }, + "dedup_dcs_abs_band": 0.5857026944751044, + "dedup_dcs_abs_band_std": 0.010728159596680428, + "matched_cos_abs_per_threshold": { + "0.5": 0.8439382115999857, + "0.6": 0.8460906744003296, + "0.7": 0.8469536105791727, + "0.8": 0.848721961180369, + "0.9": 0.8502402305603027 + }, + "matched_cos_abs_band": 0.8480016191800435, + "matched_cos_abs_band_std": 0.012445397547810365, + "unique_frac_abs_per_threshold": { + "0.5": 0.4953342013888889, + "0.6": 0.5896267361111112, + "0.7": 0.6697591145833334, + "0.8": 0.7309027777777778, + "0.9": 0.7718641493055557 + }, + "unique_frac_abs_band": 0.6905381944444445, + "unique_frac_abs_band_std": 0.003183290986805281, + "matched_frac_abs_per_threshold": { + "0.5": 0.4447157118055555, + "0.6": 0.5154622395833334, + "0.7": 0.5536024305555557, + "0.8": 0.5394422743055555, + "0.9": 0.4520399305555555 + }, + "matched_frac_abs_band": 0.51513671875, + "matched_frac_abs_band_std": 0.014130718866431944, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_128/seed0/config.json b/lm/pythia-160m_layer8/mp/x8/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b3fd244772bb48fe3d86fad8c03b0af36f99e8b2 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_128/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..970ca145ab67a3071085039d6d920c01c6b6b282 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 18.096640771484374, + "l0_per_token_0.001": 18.096640771484374, + "l0_per_token_0.01": 18.096640771484374, + "l0_per_token_0.05": 18.096640771484374, + "l0_per_token_0.1": 18.096640771484374, + "l0_per_token_0.5": 18.056124560546873, + "l0_per_token_1.0": 14.7414255859375, + "l1_per_token": 49.69892354798362, + "r2_score": 0.8900917104166631, + "l2_loss": 0.07317375053732492 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_128/seed1/config.json b/lm/pythia-160m_layer8/mp/x8/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b57334bdfb7c241cea507faaf65ed199052d3e8f --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_128/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..ce4a86435f14652cce11eb82fde9a38f5f85936d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 18.963417431640625, + "l0_per_token_0.001": 18.963417431640625, + "l0_per_token_0.01": 18.963417431640625, + "l0_per_token_0.05": 18.963417431640625, + "l0_per_token_0.1": 18.963417431640625, + "l0_per_token_0.5": 18.910694580078125, + "l0_per_token_1.0": 14.979960302734375, + "l1_per_token": 50.41817035904604, + "r2_score": 0.8912382060724657, + "l2_loss": 0.07241044699190662 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_128/seed2/config.json b/lm/pythia-160m_layer8/mp/x8/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..abd1311d294b93d49cdd3c8a13e2ddb9120dc92d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_128/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..0959fe973cce1d1bc6fad9dd94cbba738e974092 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 17.611799365234376, + "l0_per_token_0.001": 17.611799365234376, + "l0_per_token_0.01": 17.611799365234376, + "l0_per_token_0.05": 17.611799365234376, + "l0_per_token_0.1": 17.611799365234376, + "l0_per_token_0.5": 17.577349365234376, + "l0_per_token_1.0": 14.680362890625, + "l1_per_token": 49.35619929228546, + "r2_score": 0.8896896561708943, + "l2_loss": 0.07344142658973087 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_32/code_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x8/l0_32/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..70c42da87f9c964e33e4cc2b7bb7c83ac666c396 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_32/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 141.67612166666666, + "recon_r2": 0.9515022750498221, + "l1_per_token": 108.86267195249019 + }, + "seed1": { + "l0": 145.15866166666666, + "recon_r2": 0.9519063168093808, + "l1_per_token": 109.98228528452759 + }, + "seed2": { + "l0": 142.272925, + "recon_r2": 0.9519674096786727, + "l1_per_token": 109.15221582647841 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.46584384411656554, + "mcc_abs": 0.46587251649930783, + "act_iou": 0.12373119931316241, + "act_iou_n_pairs": 6144, + "input_r2": 0.9699977832547844, + "matched_mcc_signed": { + "0.6": 0.6161847228881695, + "0.7": 0.627672492947477, + "0.8": 0.6356583085944716, + "0.9": 0.6531931157131006 + }, + "matched_mcc_abs": { + "0.6": 0.6162552811127698, + "0.7": 0.626853819702229, + "0.8": 0.6365088452272449, + "0.9": 0.6530337102218212 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.4825729185943131, + "mcc_abs": 0.482592264418467, + "act_iou": 0.12935204818082505, + "act_iou_n_pairs": 6144, + "input_r2": 0.9704180507832192, + "matched_mcc_signed": { + "0.6": 0.6173570075995674, + "0.7": 0.6256302493184068, + "0.8": 0.6321348052456569, + "0.9": 0.6463717430998857 + }, + "matched_mcc_abs": { + "0.6": 0.6167434612859812, + "0.7": 0.6247887407680344, + "0.8": 0.6326111799140431, + "0.9": 0.6482760781655199 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.4694000710946156, + "mcc_abs": 0.4694255791807123, + "act_iou": 0.12436839275674312, + "act_iou_n_pairs": 6144, + "input_r2": 0.970172886562099, + "matched_mcc_signed": { + "0.6": 0.6147991360722004, + "0.7": 0.6234505420149811, + "0.8": 0.6328901849035238, + "0.9": 0.6469418418792791 + }, + "matched_mcc_abs": { + "0.6": 0.613698453513571, + "0.7": 0.6236734761961792, + "0.8": 0.6341238044623545, + "0.9": 0.647859825894199 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 143.03590277777778, + "l0_std": 1.862423831982678, + "recon_r2_mean": 0.9517920005126251, + "recon_r2_std": 0.0002527621758035025, + "l1_per_token_mean": 109.33239102116539, + "l1_per_token_std": 0.5811461338633718, + "input_r2_mean": 0.9701962402000341, + "input_r2_std": 0.0002111048155959148, + "mcc_signed_mean": 0.47260561126849804, + "mcc_signed_std": 0.008813177581059352, + "mcc_abs_mean": 0.47263012003282906, + "mcc_abs_std": 0.00880847910678798, + "matched_mcc_signed_per_threshold": { + "0.6": 0.6161136221866457, + "0.7": 0.6255844280936217, + "0.8": 0.6335610995812174, + "0.9": 0.6488355668974218 + }, + "matched_mcc_signed_band": 0.6310236791897267, + "matched_mcc_abs_per_threshold": { + "0.6": 0.615565731970774, + "0.7": 0.6251053455554808, + "0.8": 0.6344146098678809, + "0.9": 0.6497232047605134 + }, + "matched_mcc_abs_band": 0.6312022230386622, + "act_iou_mean": 0.12581721341691018, + "act_iou_std": 0.00307779084365252, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_32/dict_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x8/l0_32/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..0dec86e75d9da7257bdb41d7149ebab1821a300d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_32/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4142, + "0.6": 3246, + "0.7": 2475, + "0.8": 1880, + "0.9": 1532 + }, + "n_dup_clusters_signed": { + "0.5": 1049, + "0.6": 733, + "0.7": 450, + "0.8": 222, + "0.9": 121 + }, + "largest_dup_signed": { + "0.5": 522, + "0.6": 522, + "0.7": 494, + "0.8": 274, + "0.9": 195 + }, + "babel_mu1_signed": 0.6800978779792786, + "n_dup_abs": { + "0.5": 4160, + "0.6": 3268, + "0.7": 2495, + "0.8": 1895, + "0.9": 1551 + }, + "n_dup_clusters_abs": { + "0.5": 1053, + "0.6": 734, + "0.7": 451, + "0.8": 222, + "0.9": 126 + }, + "largest_dup_abs": { + "0.5": 604, + "0.6": 604, + "0.7": 361, + "0.8": 337, + "0.9": 195 + }, + "babel_mu1_abs": 0.6818435788154602 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4124, + "0.6": 3244, + "0.7": 2498, + "0.8": 1908, + "0.9": 1534 + }, + "n_dup_clusters_signed": { + "0.5": 1026, + "0.6": 727, + "0.7": 458, + "0.8": 235, + "0.9": 116 + }, + "largest_dup_signed": { + "0.5": 490, + "0.6": 490, + "0.7": 490, + "0.8": 352, + "0.9": 125 + }, + "babel_mu1_signed": 0.679654598236084, + "n_dup_abs": { + "0.5": 4137, + "0.6": 3270, + "0.7": 2517, + "0.8": 1921, + "0.9": 1546 + }, + "n_dup_clusters_abs": { + "0.5": 1024, + "0.6": 730, + "0.7": 459, + "0.8": 233, + "0.9": 118 + }, + "largest_dup_abs": { + "0.5": 524, + "0.6": 503, + "0.7": 499, + "0.8": 302, + "0.9": 122 + }, + "babel_mu1_abs": 0.6812613606452942 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4150, + "0.6": 3241, + "0.7": 2441, + "0.8": 1818, + "0.9": 1462 + }, + "n_dup_clusters_signed": { + "0.5": 1084, + "0.6": 771, + "0.7": 473, + "0.8": 231, + "0.9": 119 + }, + "largest_dup_signed": { + "0.5": 582, + "0.6": 399, + "0.7": 322, + "0.8": 322, + "0.9": 170 + }, + "babel_mu1_signed": 0.67706698179245, + "n_dup_abs": { + "0.5": 4169, + "0.6": 3258, + "0.7": 2458, + "0.8": 1831, + "0.9": 1475 + }, + "n_dup_clusters_abs": { + "0.5": 1087, + "0.6": 768, + "0.7": 473, + "0.8": 232, + "0.9": 123 + }, + "largest_dup_abs": { + "0.5": 584, + "0.6": 584, + "0.7": 334, + "0.8": 237, + "0.9": 170 + }, + "babel_mu1_abs": 0.6785966753959656 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8674505352973938, + "dcs_abs": 0.8728463053703308, + "frac_sign_flipped": 0.02392578125, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8343208432197571, + "0.6": 0.8349205255508423, + "0.7": 0.8357803821563721, + "0.8": 0.8370693325996399, + "0.9": 0.8390234708786011 + }, + "unique_frac_signed": { + "0.5": 0.4961751302083333, + "0.6": 0.5906575520833334, + "0.7": 0.669189453125, + "0.8": 0.7289225260416666, + "0.9": 0.769775390625 + }, + "dedup_dcs_signed": { + "0.5": 0.4139692530200894, + "0.6": 0.4931521138059907, + "0.7": 0.5592954168678261, + "0.8": 0.6101586923905415, + "0.9": 0.6458596200391185 + }, + "matched_frac_signed": { + "0.5": 0.4453125, + "0.6": 0.5128580729166666, + "0.7": 0.54443359375, + "0.8": 0.5310872395833334, + "0.9": 0.44091796875 + }, + "matched_cos_abs": { + "0.5": 0.8370665907859802, + "0.6": 0.8378727436065674, + "0.7": 0.8381679654121399, + "0.8": 0.8394980430603027, + "0.9": 0.8412030935287476 + }, + "unique_frac_abs": { + "0.5": 0.4938151041666667, + "0.6": 0.5870768229166666, + "0.7": 0.6661783854166666, + "0.8": 0.7264811197916666, + "0.9": 0.767822265625 + }, + "dedup_dcs_abs": { + "0.5": 0.4133561257234154, + "0.6": 0.49189566832501436, + "0.7": 0.5583693819062319, + "0.8": 0.6098794783853615, + "0.9": 0.6458944651240017 + }, + "matched_frac_abs": { + "0.5": 0.4446614583333333, + "0.6": 0.5120442708333334, + "0.7": 0.5458984375, + "0.8": 0.5299479166666666, + "0.9": 0.43994140625 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8815560340881348, + "dcs_abs": 0.8838018774986267, + "frac_sign_flipped": 0.019694010416666668, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8520606756210327, + "0.6": 0.8557045459747314, + "0.7": 0.8584762215614319, + "0.8": 0.860958456993103, + "0.9": 0.862807035446167 + }, + "unique_frac_signed": { + "0.5": 0.498779296875, + "0.6": 0.594482421875, + "0.7": 0.675048828125, + "0.8": 0.7359212239583334, + "0.9": 0.77587890625 + }, + "dedup_dcs_signed": { + "0.5": 0.42499022468109615, + "0.6": 0.5087013109005056, + "0.7": 0.5795133673382225, + "0.8": 0.6335976014476425, + "0.9": 0.669433778966777 + }, + "matched_frac_signed": { + "0.5": 0.4475911458333333, + "0.6": 0.51953125, + "0.7": 0.5626627604166666, + "0.8": 0.5559895833333334, + "0.9": 0.4890950520833333 + }, + "matched_cos_abs": { + "0.5": 0.8549606800079346, + "0.6": 0.8587138652801514, + "0.7": 0.8611496090888977, + "0.8": 0.8635424375534058, + "0.9": 0.8655458688735962 + }, + "unique_frac_abs": { + "0.5": 0.496337890625, + "0.6": 0.5911458333333334, + "0.7": 0.672119140625, + "0.8": 0.7337239583333334, + "0.9": 0.7740071614583334 + }, + "dedup_dcs_abs": { + "0.5": 0.42434938048245385, + "0.6": 0.5076251234859228, + "0.7": 0.5787951352103846, + "0.8": 0.6336017754705002, + "0.9": 0.669938701078839 + }, + "matched_frac_abs": { + "0.5": 0.4462890625, + "0.6": 0.5196940104166666, + "0.7": 0.5633138020833334, + "0.8": 0.5556640625, + "0.9": 0.4871419270833333 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8638496398925781, + "dcs_abs": 0.8671876788139343, + "frac_sign_flipped": 0.025227864583333332, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8361403346061707, + "0.6": 0.8372998833656311, + "0.7": 0.8382330536842346, + "0.8": 0.8397597670555115, + "0.9": 0.8406630158424377 + }, + "unique_frac_signed": { + "0.5": 0.4983723958333333, + "0.6": 0.5941569010416666, + "0.7": 0.673828125, + "0.8": 0.7347005208333334, + "0.9": 0.7753092447916666 + }, + "dedup_dcs_signed": { + "0.5": 0.4167092618105622, + "0.6": 0.4974875039430723, + "0.7": 0.5648250068770722, + "0.8": 0.616971938230563, + "0.9": 0.6517738079370853 + }, + "matched_frac_signed": { + "0.5": 0.4431966145833333, + "0.6": 0.5143229166666666, + "0.7": 0.54931640625, + "0.8": 0.5323893229166666, + "0.9": 0.4298502604166667 + }, + "matched_cos_abs": { + "0.5": 0.8397873640060425, + "0.6": 0.84168541431427, + "0.7": 0.8415432572364807, + "0.8": 0.8431254029273987, + "0.9": 0.8439717292785645 + }, + "unique_frac_abs": { + "0.5": 0.495849609375, + "0.6": 0.5906575520833334, + "0.7": 0.6709798177083334, + "0.8": 0.7325032552083334, + "0.9": 0.7737630208333334 + }, + "dedup_dcs_abs": { + "0.5": 0.4164082364004571, + "0.6": 0.497147846443113, + "0.7": 0.5646585413342109, + "0.8": 0.6175921021931572, + "0.9": 0.6530341147445142 + }, + "matched_frac_abs": { + "0.5": 0.4431966145833333, + "0.6": 0.5146484375, + "0.7": 0.5515950520833334, + "0.8": 0.53271484375, + "0.9": 0.4290364583333333 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.6789398193359375, + "babel_mu1_signed_std": 0.001636998650245021, + "n_dup_signed_per_threshold": { + "0.5": 4138.666666666667, + "0.6": 3243.6666666666665, + "0.7": 2471.3333333333335, + "0.8": 1868.6666666666667, + "0.9": 1509.3333333333333 + }, + "n_dup_signed_band": 2273.25, + "n_dup_signed_band_std": 29.069958720300928, + "n_dup_clusters_signed_per_threshold": { + "0.5": 1053.0, + "0.6": 743.6666666666666, + "0.7": 460.3333333333333, + "0.8": 229.33333333333334, + "0.9": 118.66666666666667 + }, + "n_dup_clusters_signed_band": 388.0, + "n_dup_clusters_signed_band_std": 9.17877987534291, + "largest_dup_signed_per_threshold": { + "0.5": 531.3333333333334, + "0.6": 470.3333333333333, + "0.7": 435.3333333333333, + "0.8": 316.0, + "0.9": 163.33333333333334 + }, + "largest_dup_signed_band": 346.24999999999994, + "largest_dup_signed_band_std": 37.403208418530085, + "dcs_signed_mean": 0.8709520697593689, + "dcs_signed_std": 0.009358133176195448, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.41855624650391593, + "0.6": 0.49978030954985614, + "0.7": 0.5678779303610403, + "0.8": 0.6202427440229157, + "0.9": 0.6556890689809936 + }, + "dedup_dcs_signed_band": 0.5858975132287014, + "dedup_dcs_signed_band_std": 0.010697328677874826, + "matched_cos_signed_per_threshold": { + "0.5": 0.8408406178156534, + "0.6": 0.8426416516304016, + "0.7": 0.8441632191340128, + "0.8": 0.8459291855494181, + "0.9": 0.847497840722402 + }, + "matched_cos_signed_band": 0.8450579742590586, + "matched_cos_signed_band_std": 0.012547899147488203, + "unique_frac_signed_per_threshold": { + "0.5": 0.49777560763888884, + "0.6": 0.5930989583333334, + "0.7": 0.6726888020833334, + "0.8": 0.7331814236111112, + "0.9": 0.7736545138888888 + }, + "unique_frac_signed_band": 0.6931559244791667, + "unique_frac_signed_band_std": 0.0030765459213382804, + "matched_frac_signed_per_threshold": { + "0.5": 0.44536675347222215, + "0.6": 0.5155707465277777, + "0.7": 0.5521375868055555, + "0.8": 0.5398220486111112, + "0.9": 0.4532877604166667 + }, + "matched_frac_signed_band": 0.5152045355902777, + "matched_frac_signed_band_std": 0.0143954626508884, + "babel_mu1_abs_mean": 0.68056720495224, + "babel_mu1_abs_std": 0.0017311801028579161, + "n_dup_abs_per_threshold": { + "0.5": 4155.333333333333, + "0.6": 3265.3333333333335, + "0.7": 2490.0, + "0.8": 1882.3333333333333, + "0.9": 1524.0 + }, + "n_dup_abs_band": 2290.416666666667, + "n_dup_abs_band_std": 30.757451671641025, + "n_dup_clusters_abs_per_threshold": { + "0.5": 1054.6666666666667, + "0.6": 744.0, + "0.7": 461.0, + "0.8": 229.0, + "0.9": 122.33333333333333 + }, + "n_dup_clusters_abs_band": 389.0833333333333, + "n_dup_clusters_abs_band_std": 8.632545009053434, + "largest_dup_abs_per_threshold": { + "0.5": 570.6666666666666, + "0.6": 563.6666666666666, + "0.7": 398.0, + "0.8": 292.0, + "0.9": 162.33333333333334 + }, + "largest_dup_abs_band": 353.99999999999994, + "largest_dup_abs_band_std": 21.608736659046034, + "dcs_abs_mean": 0.8746119538942972, + "dcs_abs_std": 0.008446658245498468, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.41803791420210884, + "0.6": 0.4988895460846834, + "0.7": 0.5672743528169425, + "0.8": 0.620357785349673, + "0.9": 0.6562890936491184 + }, + "dedup_dcs_abs_band": 0.5857026944751044, + "dedup_dcs_abs_band_std": 0.010728159596680428, + "matched_cos_abs_per_threshold": { + "0.5": 0.8439382115999857, + "0.6": 0.8460906744003296, + "0.7": 0.8469536105791727, + "0.8": 0.848721961180369, + "0.9": 0.8502402305603027 + }, + "matched_cos_abs_band": 0.8480016191800435, + "matched_cos_abs_band_std": 0.012445397547810365, + "unique_frac_abs_per_threshold": { + "0.5": 0.4953342013888889, + "0.6": 0.5896267361111112, + "0.7": 0.6697591145833334, + "0.8": 0.7309027777777778, + "0.9": 0.7718641493055557 + }, + "unique_frac_abs_band": 0.6905381944444445, + "unique_frac_abs_band_std": 0.003183290986805281, + "matched_frac_abs_per_threshold": { + "0.5": 0.4447157118055555, + "0.6": 0.5154622395833334, + "0.7": 0.5536024305555557, + "0.8": 0.5394422743055555, + "0.9": 0.4520399305555555 + }, + "matched_frac_abs_band": 0.51513671875, + "matched_frac_abs_band_std": 0.014130718866431944, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_32/seed0/config.json b/lm/pythia-160m_layer8/mp/x8/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b3fd244772bb48fe3d86fad8c03b0af36f99e8b2 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_32/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..970ca145ab67a3071085039d6d920c01c6b6b282 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 18.096640771484374, + "l0_per_token_0.001": 18.096640771484374, + "l0_per_token_0.01": 18.096640771484374, + "l0_per_token_0.05": 18.096640771484374, + "l0_per_token_0.1": 18.096640771484374, + "l0_per_token_0.5": 18.056124560546873, + "l0_per_token_1.0": 14.7414255859375, + "l1_per_token": 49.69892354798362, + "r2_score": 0.8900917104166631, + "l2_loss": 0.07317375053732492 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_32/seed1/config.json b/lm/pythia-160m_layer8/mp/x8/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b57334bdfb7c241cea507faaf65ed199052d3e8f --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_32/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..ce4a86435f14652cce11eb82fde9a38f5f85936d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 18.963417431640625, + "l0_per_token_0.001": 18.963417431640625, + "l0_per_token_0.01": 18.963417431640625, + "l0_per_token_0.05": 18.963417431640625, + "l0_per_token_0.1": 18.963417431640625, + "l0_per_token_0.5": 18.910694580078125, + "l0_per_token_1.0": 14.979960302734375, + "l1_per_token": 50.41817035904604, + "r2_score": 0.8912382060724657, + "l2_loss": 0.07241044699190662 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_32/seed2/config.json b/lm/pythia-160m_layer8/mp/x8/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..abd1311d294b93d49cdd3c8a13e2ddb9120dc92d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_32/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..0959fe973cce1d1bc6fad9dd94cbba738e974092 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 17.611799365234376, + "l0_per_token_0.001": 17.611799365234376, + "l0_per_token_0.01": 17.611799365234376, + "l0_per_token_0.05": 17.611799365234376, + "l0_per_token_0.1": 17.611799365234376, + "l0_per_token_0.5": 17.577349365234376, + "l0_per_token_1.0": 14.680362890625, + "l1_per_token": 49.35619929228546, + "r2_score": 0.8896896561708943, + "l2_loss": 0.07344142658973087 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/code_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x8/l0_64/code_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..70c42da87f9c964e33e4cc2b7bb7c83ac666c396 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/code_metrics_step_final.json @@ -0,0 +1,136 @@ +{ + "schema": 1, + "step": "final", + "n_tokens": 600000, + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "l0": 141.67612166666666, + "recon_r2": 0.9515022750498221, + "l1_per_token": 108.86267195249019 + }, + "seed1": { + "l0": 145.15866166666666, + "recon_r2": 0.9519063168093808, + "l1_per_token": 109.98228528452759 + }, + "seed2": { + "l0": 142.272925, + "recon_r2": 0.9519674096786727, + "l1_per_token": 109.15221582647841 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "alive_source": "stats", + "mcc_signed": 0.46584384411656554, + "mcc_abs": 0.46587251649930783, + "act_iou": 0.12373119931316241, + "act_iou_n_pairs": 6144, + "input_r2": 0.9699977832547844, + "matched_mcc_signed": { + "0.6": 0.6161847228881695, + "0.7": 0.627672492947477, + "0.8": 0.6356583085944716, + "0.9": 0.6531931157131006 + }, + "matched_mcc_abs": { + "0.6": 0.6162552811127698, + "0.7": 0.626853819702229, + "0.8": 0.6365088452272449, + "0.9": 0.6530337102218212 + } + }, + "seed0\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.4825729185943131, + "mcc_abs": 0.482592264418467, + "act_iou": 0.12935204818082505, + "act_iou_n_pairs": 6144, + "input_r2": 0.9704180507832192, + "matched_mcc_signed": { + "0.6": 0.6173570075995674, + "0.7": 0.6256302493184068, + "0.8": 0.6321348052456569, + "0.9": 0.6463717430998857 + }, + "matched_mcc_abs": { + "0.6": 0.6167434612859812, + "0.7": 0.6247887407680344, + "0.8": 0.6326111799140431, + "0.9": 0.6482760781655199 + } + }, + "seed1\u2194seed2": { + "alive_source": "stats", + "mcc_signed": 0.4694000710946156, + "mcc_abs": 0.4694255791807123, + "act_iou": 0.12436839275674312, + "act_iou_n_pairs": 6144, + "input_r2": 0.970172886562099, + "matched_mcc_signed": { + "0.6": 0.6147991360722004, + "0.7": 0.6234505420149811, + "0.8": 0.6328901849035238, + "0.9": 0.6469418418792791 + }, + "matched_mcc_abs": { + "0.6": 0.613698453513571, + "0.7": 0.6236734761961792, + "0.8": 0.6341238044623545, + "0.9": 0.647859825894199 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "l0_mean": 143.03590277777778, + "l0_std": 1.862423831982678, + "recon_r2_mean": 0.9517920005126251, + "recon_r2_std": 0.0002527621758035025, + "l1_per_token_mean": 109.33239102116539, + "l1_per_token_std": 0.5811461338633718, + "input_r2_mean": 0.9701962402000341, + "input_r2_std": 0.0002111048155959148, + "mcc_signed_mean": 0.47260561126849804, + "mcc_signed_std": 0.008813177581059352, + "mcc_abs_mean": 0.47263012003282906, + "mcc_abs_std": 0.00880847910678798, + "matched_mcc_signed_per_threshold": { + "0.6": 0.6161136221866457, + "0.7": 0.6255844280936217, + "0.8": 0.6335610995812174, + "0.9": 0.6488355668974218 + }, + "matched_mcc_signed_band": 0.6310236791897267, + "matched_mcc_abs_per_threshold": { + "0.6": 0.615565731970774, + "0.7": 0.6251053455554808, + "0.8": 0.6344146098678809, + "0.9": 0.6497232047605134 + }, + "matched_mcc_abs_band": 0.6312022230386622, + "act_iou_mean": 0.12581721341691018, + "act_iou_std": 0.00307779084365252, + "act_iou_n_pairs": 6144.0, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/dict_metrics_step_final.json b/lm/pythia-160m_layer8/mp/x8/l0_64/dict_metrics_step_final.json new file mode 100644 index 0000000000000000000000000000000000000000..0dec86e75d9da7257bdb41d7149ebab1821a300d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/dict_metrics_step_final.json @@ -0,0 +1,545 @@ +{ + "schema": 2, + "step": "final", + "linkage": "complete", + "thresholds": [ + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "seeds": [ + "seed0", + "seed1", + "seed2" + ], + "per_seed": { + "seed0": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4142, + "0.6": 3246, + "0.7": 2475, + "0.8": 1880, + "0.9": 1532 + }, + "n_dup_clusters_signed": { + "0.5": 1049, + "0.6": 733, + "0.7": 450, + "0.8": 222, + "0.9": 121 + }, + "largest_dup_signed": { + "0.5": 522, + "0.6": 522, + "0.7": 494, + "0.8": 274, + "0.9": 195 + }, + "babel_mu1_signed": 0.6800978779792786, + "n_dup_abs": { + "0.5": 4160, + "0.6": 3268, + "0.7": 2495, + "0.8": 1895, + "0.9": 1551 + }, + "n_dup_clusters_abs": { + "0.5": 1053, + "0.6": 734, + "0.7": 451, + "0.8": 222, + "0.9": 126 + }, + "largest_dup_abs": { + "0.5": 604, + "0.6": 604, + "0.7": 361, + "0.8": 337, + "0.9": 195 + }, + "babel_mu1_abs": 0.6818435788154602 + }, + "seed1": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4124, + "0.6": 3244, + "0.7": 2498, + "0.8": 1908, + "0.9": 1534 + }, + "n_dup_clusters_signed": { + "0.5": 1026, + "0.6": 727, + "0.7": 458, + "0.8": 235, + "0.9": 116 + }, + "largest_dup_signed": { + "0.5": 490, + "0.6": 490, + "0.7": 490, + "0.8": 352, + "0.9": 125 + }, + "babel_mu1_signed": 0.679654598236084, + "n_dup_abs": { + "0.5": 4137, + "0.6": 3270, + "0.7": 2517, + "0.8": 1921, + "0.9": 1546 + }, + "n_dup_clusters_abs": { + "0.5": 1024, + "0.6": 730, + "0.7": 459, + "0.8": 233, + "0.9": 118 + }, + "largest_dup_abs": { + "0.5": 524, + "0.6": 503, + "0.7": 499, + "0.8": 302, + "0.9": 122 + }, + "babel_mu1_abs": 0.6812613606452942 + }, + "seed2": { + "d_sae": 6144, + "d_in": 768, + "l0": null, + "l0_raw": null, + "recon_r2": null, + "l1_per_token": null, + "stats_source": "mp-excluded", + "n_dead": 0, + "dead_rate": 0.0, + "alive_source": "stats", + "n_dense": 0, + "dense_rate": 0.0, + "dense_threshold": 0.1, + "n_dup_signed": { + "0.5": 4150, + "0.6": 3241, + "0.7": 2441, + "0.8": 1818, + "0.9": 1462 + }, + "n_dup_clusters_signed": { + "0.5": 1084, + "0.6": 771, + "0.7": 473, + "0.8": 231, + "0.9": 119 + }, + "largest_dup_signed": { + "0.5": 582, + "0.6": 399, + "0.7": 322, + "0.8": 322, + "0.9": 170 + }, + "babel_mu1_signed": 0.67706698179245, + "n_dup_abs": { + "0.5": 4169, + "0.6": 3258, + "0.7": 2458, + "0.8": 1831, + "0.9": 1475 + }, + "n_dup_clusters_abs": { + "0.5": 1087, + "0.6": 768, + "0.7": 473, + "0.8": 232, + "0.9": 123 + }, + "largest_dup_abs": { + "0.5": 584, + "0.6": 584, + "0.7": 334, + "0.8": 237, + "0.9": 170 + }, + "babel_mu1_abs": 0.6785966753959656 + } + }, + "per_pair": { + "seed0\u2194seed1": { + "dcs_signed": 0.8674505352973938, + "dcs_abs": 0.8728463053703308, + "frac_sign_flipped": 0.02392578125, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8343208432197571, + "0.6": 0.8349205255508423, + "0.7": 0.8357803821563721, + "0.8": 0.8370693325996399, + "0.9": 0.8390234708786011 + }, + "unique_frac_signed": { + "0.5": 0.4961751302083333, + "0.6": 0.5906575520833334, + "0.7": 0.669189453125, + "0.8": 0.7289225260416666, + "0.9": 0.769775390625 + }, + "dedup_dcs_signed": { + "0.5": 0.4139692530200894, + "0.6": 0.4931521138059907, + "0.7": 0.5592954168678261, + "0.8": 0.6101586923905415, + "0.9": 0.6458596200391185 + }, + "matched_frac_signed": { + "0.5": 0.4453125, + "0.6": 0.5128580729166666, + "0.7": 0.54443359375, + "0.8": 0.5310872395833334, + "0.9": 0.44091796875 + }, + "matched_cos_abs": { + "0.5": 0.8370665907859802, + "0.6": 0.8378727436065674, + "0.7": 0.8381679654121399, + "0.8": 0.8394980430603027, + "0.9": 0.8412030935287476 + }, + "unique_frac_abs": { + "0.5": 0.4938151041666667, + "0.6": 0.5870768229166666, + "0.7": 0.6661783854166666, + "0.8": 0.7264811197916666, + "0.9": 0.767822265625 + }, + "dedup_dcs_abs": { + "0.5": 0.4133561257234154, + "0.6": 0.49189566832501436, + "0.7": 0.5583693819062319, + "0.8": 0.6098794783853615, + "0.9": 0.6458944651240017 + }, + "matched_frac_abs": { + "0.5": 0.4446614583333333, + "0.6": 0.5120442708333334, + "0.7": 0.5458984375, + "0.8": 0.5299479166666666, + "0.9": 0.43994140625 + } + }, + "seed0\u2194seed2": { + "dcs_signed": 0.8815560340881348, + "dcs_abs": 0.8838018774986267, + "frac_sign_flipped": 0.019694010416666668, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8520606756210327, + "0.6": 0.8557045459747314, + "0.7": 0.8584762215614319, + "0.8": 0.860958456993103, + "0.9": 0.862807035446167 + }, + "unique_frac_signed": { + "0.5": 0.498779296875, + "0.6": 0.594482421875, + "0.7": 0.675048828125, + "0.8": 0.7359212239583334, + "0.9": 0.77587890625 + }, + "dedup_dcs_signed": { + "0.5": 0.42499022468109615, + "0.6": 0.5087013109005056, + "0.7": 0.5795133673382225, + "0.8": 0.6335976014476425, + "0.9": 0.669433778966777 + }, + "matched_frac_signed": { + "0.5": 0.4475911458333333, + "0.6": 0.51953125, + "0.7": 0.5626627604166666, + "0.8": 0.5559895833333334, + "0.9": 0.4890950520833333 + }, + "matched_cos_abs": { + "0.5": 0.8549606800079346, + "0.6": 0.8587138652801514, + "0.7": 0.8611496090888977, + "0.8": 0.8635424375534058, + "0.9": 0.8655458688735962 + }, + "unique_frac_abs": { + "0.5": 0.496337890625, + "0.6": 0.5911458333333334, + "0.7": 0.672119140625, + "0.8": 0.7337239583333334, + "0.9": 0.7740071614583334 + }, + "dedup_dcs_abs": { + "0.5": 0.42434938048245385, + "0.6": 0.5076251234859228, + "0.7": 0.5787951352103846, + "0.8": 0.6336017754705002, + "0.9": 0.669938701078839 + }, + "matched_frac_abs": { + "0.5": 0.4462890625, + "0.6": 0.5196940104166666, + "0.7": 0.5633138020833334, + "0.8": 0.5556640625, + "0.9": 0.4871419270833333 + } + }, + "seed1\u2194seed2": { + "dcs_signed": 0.8638496398925781, + "dcs_abs": 0.8671876788139343, + "frac_sign_flipped": 0.025227864583333332, + "alive_source": "stats", + "n_alive_a": 6144, + "n_alive_b": 6144, + "matched_cos_signed": { + "0.5": 0.8361403346061707, + "0.6": 0.8372998833656311, + "0.7": 0.8382330536842346, + "0.8": 0.8397597670555115, + "0.9": 0.8406630158424377 + }, + "unique_frac_signed": { + "0.5": 0.4983723958333333, + "0.6": 0.5941569010416666, + "0.7": 0.673828125, + "0.8": 0.7347005208333334, + "0.9": 0.7753092447916666 + }, + "dedup_dcs_signed": { + "0.5": 0.4167092618105622, + "0.6": 0.4974875039430723, + "0.7": 0.5648250068770722, + "0.8": 0.616971938230563, + "0.9": 0.6517738079370853 + }, + "matched_frac_signed": { + "0.5": 0.4431966145833333, + "0.6": 0.5143229166666666, + "0.7": 0.54931640625, + "0.8": 0.5323893229166666, + "0.9": 0.4298502604166667 + }, + "matched_cos_abs": { + "0.5": 0.8397873640060425, + "0.6": 0.84168541431427, + "0.7": 0.8415432572364807, + "0.8": 0.8431254029273987, + "0.9": 0.8439717292785645 + }, + "unique_frac_abs": { + "0.5": 0.495849609375, + "0.6": 0.5906575520833334, + "0.7": 0.6709798177083334, + "0.8": 0.7325032552083334, + "0.9": 0.7737630208333334 + }, + "dedup_dcs_abs": { + "0.5": 0.4164082364004571, + "0.6": 0.497147846443113, + "0.7": 0.5646585413342109, + "0.8": 0.6175921021931572, + "0.9": 0.6530341147445142 + }, + "matched_frac_abs": { + "0.5": 0.4431966145833333, + "0.6": 0.5146484375, + "0.7": 0.5515950520833334, + "0.8": 0.53271484375, + "0.9": 0.4290364583333333 + } + } + }, + "summary": { + "n_seeds": 3, + "n_pairs": 3, + "report_band": [ + 0.6, + 0.7, + 0.8, + 0.9 + ], + "n_dead_mean": 0, + "n_dead_std": 0.0, + "n_dense_mean": 0, + "n_dense_std": 0.0, + "l0_mean": null, + "l0_std": null, + "recon_r2_mean": null, + "recon_r2_std": null, + "l1_per_token_mean": null, + "l1_per_token_std": null, + "babel_mu1_signed_mean": 0.6789398193359375, + "babel_mu1_signed_std": 0.001636998650245021, + "n_dup_signed_per_threshold": { + "0.5": 4138.666666666667, + "0.6": 3243.6666666666665, + "0.7": 2471.3333333333335, + "0.8": 1868.6666666666667, + "0.9": 1509.3333333333333 + }, + "n_dup_signed_band": 2273.25, + "n_dup_signed_band_std": 29.069958720300928, + "n_dup_clusters_signed_per_threshold": { + "0.5": 1053.0, + "0.6": 743.6666666666666, + "0.7": 460.3333333333333, + "0.8": 229.33333333333334, + "0.9": 118.66666666666667 + }, + "n_dup_clusters_signed_band": 388.0, + "n_dup_clusters_signed_band_std": 9.17877987534291, + "largest_dup_signed_per_threshold": { + "0.5": 531.3333333333334, + "0.6": 470.3333333333333, + "0.7": 435.3333333333333, + "0.8": 316.0, + "0.9": 163.33333333333334 + }, + "largest_dup_signed_band": 346.24999999999994, + "largest_dup_signed_band_std": 37.403208418530085, + "dcs_signed_mean": 0.8709520697593689, + "dcs_signed_std": 0.009358133176195448, + "dedup_dcs_signed_per_threshold": { + "0.5": 0.41855624650391593, + "0.6": 0.49978030954985614, + "0.7": 0.5678779303610403, + "0.8": 0.6202427440229157, + "0.9": 0.6556890689809936 + }, + "dedup_dcs_signed_band": 0.5858975132287014, + "dedup_dcs_signed_band_std": 0.010697328677874826, + "matched_cos_signed_per_threshold": { + "0.5": 0.8408406178156534, + "0.6": 0.8426416516304016, + "0.7": 0.8441632191340128, + "0.8": 0.8459291855494181, + "0.9": 0.847497840722402 + }, + "matched_cos_signed_band": 0.8450579742590586, + "matched_cos_signed_band_std": 0.012547899147488203, + "unique_frac_signed_per_threshold": { + "0.5": 0.49777560763888884, + "0.6": 0.5930989583333334, + "0.7": 0.6726888020833334, + "0.8": 0.7331814236111112, + "0.9": 0.7736545138888888 + }, + "unique_frac_signed_band": 0.6931559244791667, + "unique_frac_signed_band_std": 0.0030765459213382804, + "matched_frac_signed_per_threshold": { + "0.5": 0.44536675347222215, + "0.6": 0.5155707465277777, + "0.7": 0.5521375868055555, + "0.8": 0.5398220486111112, + "0.9": 0.4532877604166667 + }, + "matched_frac_signed_band": 0.5152045355902777, + "matched_frac_signed_band_std": 0.0143954626508884, + "babel_mu1_abs_mean": 0.68056720495224, + "babel_mu1_abs_std": 0.0017311801028579161, + "n_dup_abs_per_threshold": { + "0.5": 4155.333333333333, + "0.6": 3265.3333333333335, + "0.7": 2490.0, + "0.8": 1882.3333333333333, + "0.9": 1524.0 + }, + "n_dup_abs_band": 2290.416666666667, + "n_dup_abs_band_std": 30.757451671641025, + "n_dup_clusters_abs_per_threshold": { + "0.5": 1054.6666666666667, + "0.6": 744.0, + "0.7": 461.0, + "0.8": 229.0, + "0.9": 122.33333333333333 + }, + "n_dup_clusters_abs_band": 389.0833333333333, + "n_dup_clusters_abs_band_std": 8.632545009053434, + "largest_dup_abs_per_threshold": { + "0.5": 570.6666666666666, + "0.6": 563.6666666666666, + "0.7": 398.0, + "0.8": 292.0, + "0.9": 162.33333333333334 + }, + "largest_dup_abs_band": 353.99999999999994, + "largest_dup_abs_band_std": 21.608736659046034, + "dcs_abs_mean": 0.8746119538942972, + "dcs_abs_std": 0.008446658245498468, + "dedup_dcs_abs_per_threshold": { + "0.5": 0.41803791420210884, + "0.6": 0.4988895460846834, + "0.7": 0.5672743528169425, + "0.8": 0.620357785349673, + "0.9": 0.6562890936491184 + }, + "dedup_dcs_abs_band": 0.5857026944751044, + "dedup_dcs_abs_band_std": 0.010728159596680428, + "matched_cos_abs_per_threshold": { + "0.5": 0.8439382115999857, + "0.6": 0.8460906744003296, + "0.7": 0.8469536105791727, + "0.8": 0.848721961180369, + "0.9": 0.8502402305603027 + }, + "matched_cos_abs_band": 0.8480016191800435, + "matched_cos_abs_band_std": 0.012445397547810365, + "unique_frac_abs_per_threshold": { + "0.5": 0.4953342013888889, + "0.6": 0.5896267361111112, + "0.7": 0.6697591145833334, + "0.8": 0.7309027777777778, + "0.9": 0.7718641493055557 + }, + "unique_frac_abs_band": 0.6905381944444445, + "unique_frac_abs_band_std": 0.003183290986805281, + "matched_frac_abs_per_threshold": { + "0.5": 0.4447157118055555, + "0.6": 0.5154622395833334, + "0.7": 0.5536024305555557, + "0.8": 0.5394422743055555, + "0.9": 0.4520399305555555 + }, + "matched_frac_abs_band": 0.51513671875, + "matched_frac_abs_band_std": 0.014130718866431944, + "alive_source": "stats" + } +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/seed0/config.json b/lm/pythia-160m_layer8/mp/x8/l0_64/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b3fd244772bb48fe3d86fad8c03b0af36f99e8b2 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/seed0/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..970ca145ab67a3071085039d6d920c01c6b6b282 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 18.096640771484374, + "l0_per_token_0.001": 18.096640771484374, + "l0_per_token_0.01": 18.096640771484374, + "l0_per_token_0.05": 18.096640771484374, + "l0_per_token_0.1": 18.096640771484374, + "l0_per_token_0.5": 18.056124560546873, + "l0_per_token_1.0": 14.7414255859375, + "l1_per_token": 49.69892354798362, + "r2_score": 0.8900917104166631, + "l2_loss": 0.07317375053732492 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/config.json b/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b57334bdfb7c241cea507faaf65ed199052d3e8f --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/model.safetensors b/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8b181993d57775f5578508e962b71e68c8ffd56c --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71f3b96c1a21125185a9b119b50e76511f5dbd408c625d3349ff6edc6e57572f +size 37776696 diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..36c63246b2ae223845ed4e89f64455e133443738 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6c749828d6f22fc58a3475b22859c67b8f82121694433253aa46ea46ceb7e54 +size 50496 diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..834c56b452e150e660d09415db7e026bc28b7d68 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90077ff318bc0b5b7d7252049b1582c06f7c79ff8d3c631dcf44ddca98b88ba7 +size 25895 diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..ce4a86435f14652cce11eb82fde9a38f5f85936d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 18.963417431640625, + "l0_per_token_0.001": 18.963417431640625, + "l0_per_token_0.01": 18.963417431640625, + "l0_per_token_0.05": 18.963417431640625, + "l0_per_token_0.1": 18.963417431640625, + "l0_per_token_0.5": 18.910694580078125, + "l0_per_token_1.0": 14.979960302734375, + "l1_per_token": 50.41817035904604, + "r2_score": 0.8912382060724657, + "l2_loss": 0.07241044699190662 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/seed2/config.json b/lm/pythia-160m_layer8/mp/x8/l0_64/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..abd1311d294b93d49cdd3c8a13e2ddb9120dc92d --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/seed2/config.json @@ -0,0 +1,18 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "mp", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "k_iters": 10, + "mp_nonneg": true, + "mp_dropout": 0.0, + "inference_mode": "adaptive", + "inference_iters": 10, + "residual_thr": 0.05, + "max_iters": 614, + "patience": 2, + "class_type": "MPSAE", + "module_name": "consdl.models.mp" +} diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..0959fe973cce1d1bc6fad9dd94cbba738e974092 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 17.611799365234376, + "l0_per_token_0.001": 17.611799365234376, + "l0_per_token_0.01": 17.611799365234376, + "l0_per_token_0.05": 17.611799365234376, + "l0_per_token_0.1": 17.611799365234376, + "l0_per_token_0.5": 17.577349365234376, + "l0_per_token_1.0": 14.680362890625, + "l1_per_token": 49.35619929228546, + "r2_score": 0.8896896561708943, + "l2_loss": 0.07344142658973087 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/mp/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/model.safetensors b/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..23cd4ec132b617fb558e5075b26cc3f71c91f592 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:040edef4d993339fa690e0b72f8d718f532fb6b924be39715ac7e8f8ce9f7413 +size 75550008 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..18d2912949bfdc7212b2d9dd9267e40d79bc1b48 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33ed3a4adc4b11f7d99b7a777d72d94641d8107f75192737f1d738f641ac7cc3 +size 99648 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9d5b025d51f178156cca90f1503866a5b67893e --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a055cecf2722ecf337ef31855faf3827e53cf2c34206c50b377c365ee6cf09cb +size 50471 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c1edaa7b9ed89382b5296279e5c9178213a8391 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0409ccb9702c3da3d2de844d1f2d7fb361369d1cc8157daffb23125f8801242f +size 50382 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e5bdf874f4f818204ce8fb52ef0bc77c76a4d329 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59a87db8165fa4176982f4e639168e27d31f18667a95ded2e717e26e15f47ebc +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fdff39aee786aeb48d3820282ee6f10596c3d2f --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e13d015295f729238f79f089a226c212075ae1f96fb6a6c6fcc27b2452de4bd6 +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/model.safetensors b/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c04c5302cc2565621a50c33dda3e78a9282fa06c --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b50c615c5615f73c6b543c134d54350e89e4b036c615cfcc6c3b455a866a00af +size 75550008 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..39f73a5390a83bf3046836eb4cd373dc1532f61b --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:966ac351587e2380981e03405a66b52eda3ea28ce8486cd5f08a953abbc158ca +size 99648 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a06f02fb5230204ea38a8b1090f374274adccee --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16e0ca06914822f03294f1872c8dada97adea9bc76e649245df8b73ac449b148 +size 50471 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..20dad40b0a5b05a88bea9f5af24be5787cf7eda5 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fad39cefe4ccbb876bb53b7ed9400ece61fe25d3c3c19db2b30b0f75b6f93994 +size 50382 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..25d1deaad68b587efb9be271eeb8076261a1e9dc --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8b4d48ef5753eadce3e4853c0578196329bc531d0f8efc97401eb1cda972252 +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b28a5b660fe35784a50b3ced9e72747abf949ae --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:977bd939c1796e647e395b68ef19440af62fffc32c24a17a08b41498a2d1e17b +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/model.safetensors b/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fd589b14f47d746a10e35801f4c2f9753a5efe87 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3974fd6743ed308cb260ad80a90dbfbb69794fc493caa954d14864bda59fb268 +size 75550008 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c3f494722fc6f54a13087e659b6688f2f0043d5 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c41a2d40bef52707b3f84f96d628b0239c47123451e99f9fa380272a9aa460f +size 99648 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..e620c2eeba7dfb38dae190cffea0ad13ee9e7eb8 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c1904b3cfbd8a3b7aab1d61a67dfbf3fec6ad48a3162d51d164aba0edfb576e +size 50471 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..46d96af070a3f7065b76c541ae173f35475f13bd --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fef9918cf6f5d9c0131dde79060c3c29abfe0ff7060870243c74dc24096bfc1a +size 50382 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..740ba31a3c32733e8147ad1b22ee4fb2becb6b54 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a5f811c1b924b09ceed5dfef27fbd7b4069644902a91d4828a95eb16816d000 +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5834cfad3a59964d0af8fbeca9a6f92171219030 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5395edf004b0a8063942079a910071d28b7050550d6d925dbfe827a574fc0065 +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/model.safetensors b/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cd4a52878c139dd37ea870021162f2f61e14cbda --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3030e6dfb33ef10b3c30542af2068cbca5606496bd1aad955fa341d1bdb2bd4 +size 75550008 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..573f845f5d1e60a9ad9d1a04a951e3adb890b483 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed080ec1e6a3d1612ee49b02a5ced92759582dcfeb3e460b1b705adb7b515a17 +size 99648 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f1e0202cdcdbe91cf7913b60c25e066acbfb521 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38a364392fd87ea02d2e417ee02fccbbd4799268ef6cda5e5f452188b948b36a +size 50471 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..81ae2c3e265ebb25f28f8dba923560dd147dc3f3 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cf9055be0d20c0a1a5d58943af7123ce67c3af256841978587b73e280ae9ce4 +size 50382 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..06eab469923afb68c9e99df677068ea6579cc734 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53ba00a4849972b2a19ac36b8dafbc499943bfa8329d847e290d741f4412f32d +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..de20846f11c0de6668f108bfa99c6ca7a2f3e7c1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9aba7b59b71001131fc804aad933d59a30aa49967c14696324f2b46fc9cf861 +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/model.safetensors b/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d4cd223f8c0d410abab4e58ba2fbda10663e7dd9 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70edb6867b6ab8404b171514acd00ae1896091141462aa474825905533ab93f8 +size 75550008 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5db750f3a426e2b8c15148d07fe08b1a1201739c --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a361fc8461d1feab4534c22168f3d8dfe081f63530da6fe295561a4584a14a3 +size 99648 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a0570edd0adbecea3aa13081afdac0a7d000fa9 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0808765ab480233e13a3f5a876d6590491d9c49fb4cc79ba606fdf9879eba03e +size 50471 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2824c030f59ff2263ae43ae2ea2458ef931860e --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cc77c8007dd802c8b0d61bf30fee60b5b9c54f601e44535398fb920771f177c +size 50382 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c55b4df694d045fec3272937efd58972f15041f --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f17463d90df1dec7c1c3e5e00e73eb8c1c48292de6c6baf5394a5ffbaf68d7d5 +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..41b5fb53a7a7a35bb84fc282ca3cc1314abbb323 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b56c547a8d6ab320e965f9aec857dfd85a61e21aa86e0ab798afdee2cc5a0e49 +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/model.safetensors b/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1d04b0db134d651acc811e823b0395e1282c9002 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f131d327f094d804ee5c5217cfa17c70ea6bdbd891c37df8e983ea15a904fe81 +size 75550008 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d22f584ce65de5ab79a90fa589fe5ad1aae834d --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8da7070271fb2ec10a4130eff36f3527e8c14322f86b1d088c94742941c5f54c +size 99648 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae6d0004dbc6bac986fbdcdee19059df68457b4b --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76859c3662646d5959ff5644c87df9e61c2e8f16aed050e5e25db686cfcfc23d +size 50471 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..18ff4daf865e9e2a4870a976799d0dc9f677b810 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0ad3e6543dbac7b79dbb7fbfd2064c855fa5653c3e32ef2ba1908798aaf3f65 +size 50382 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba3537683e4f4ab956b842e2d869e919b1b5be22 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:215ba210d9ee2fa29d423082c2f52ec1d4581405d399ac36dff8c93b53bc930e +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2d92ed33642969eec7b11d0445a76c2a67d85e7 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17f46847294354327699876f31d21aca71aaefa4fa55d82374c829482b2ff772 +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/model.safetensors b/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f982709224d029b306238190058d896c2bdb3d6d --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b916fcb1f3bf83eea915c166638112117ecbd34472f129e72042e6095387eb77 +size 75550008 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba87ea7b9a3dabd21a5bed4bdb2cc90c0833ad25 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebc2798e5eabaeb610a3f3110e525e394183f835fd99abd5bf6cc0857fff74cb +size 99648 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2dcf2dd378eaa5875eac5f5d130c0aa699f6362 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed3caf12aa305719984746a42954c6e5c706e1cb076b0b262d586327489f8d18 +size 50471 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b25ca1a17c1869092374c2aee6c3ae7c7318ce3 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ae27fde6e218058b9c000c5600e5fdd494b21219d1ab4c06bd41087a816a166 +size 50382 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..32fc692b652d8bd5c55cf75b51475242e99a076b --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc53375c4de7650d792ca85a62b62a6206edb916a2225e2c392d682ffcb0f8b5 +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..75d9ee275a344e86bf4aacbd8b723d2cc4f89e62 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36b35f75c5db27f9cdbb2288a5a36d3127bb8f3c8174c199c0c796c32761f8a5 +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/model.safetensors b/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0ec9591c3be9cf9bdbe6f87122a83472887a4a23 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60eab5e6279635d4400208a63f79c114e13e4598db908b5862f87c5692fa969e +size 75550008 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b74702c2cb47ef300f244d128d3e58fb475f3dec --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6bbdd20c05d698d025737166fb4fc477626f32931b0e27a4ddab41a0abbeccd +size 99648 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9e8f1165020759b5d191f70fde7a7438a173781 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0c00537f72c13fb9944cb59d86e7abeda59aa4c15797b47d283196e69d08680 +size 50471 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e10c9698a75adb9f07af1e5ae8b9dcef1a4075c2 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f2a6264b30f591a69ad552f2f7792a4f5fd9aaad3618314bde89fabf48f8b9a +size 50382 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5b2a1559bead753291de3a3630c0b556aac6285 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4e3c11ae12ba44467baba2078270ee88324bb683eca4c7425193bd0b6dee757 +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c5996416381073e9719d2f712f3ea7271a364ad --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21bb33a6a9ccdb788d436f2e36c5a733dc2a5b9fe415feeea10c5b5fafaed1ae +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/model.safetensors b/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4163ac59f9f0fda6e9730dce38b53b36dcec1f3d --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:668ad055640e67919e2b20f7ba820386c17ca857e8963a609c38fc172146f6e4 +size 75550008 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..715ac7ac00e2ccf826ea0e10b62bed22e9fba83a --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b0fd257d5005d7e4c83c2030ff8273425ae8d19aebb6ec4ad74f507aad5600c +size 99648 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..faec6e19e399700e15fd1e9c28b4fd08988505ac --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83ddc923d285306e1eebcce92f75b1aa43d022ca16c0200c05a9ba110c2efa47 +size 50471 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfe0281340c2d4b9fb1f20ee331a4d3080c28765 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbfb6d274b11ef12617af69daaf407378e23e9d626e922bde8eac1a11f816498 +size 50382 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a29e23dbca882d93860181813a6c5bde982faea --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8a8bf8c4ae6b41ad8cc7b217bd809dd578785922a4f170b18dfc2245a829762 +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc3b1ddc879be1ea7eec6182b7bfa01e44d38581 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0dfb40db982540077a19d23df5fab36b7bc920ffd7174a9ff94f120881160f9 +size 50387 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/model.safetensors b/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d48fa54b37f6c8a3c7b454ce94440f5453ce0688 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d90238490012d6b4bfdba5433eb72c603f0ab61f289724a5ef18439fc3295606 +size 18890032 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d2ca7104ad083bfe3571fc78dd99d566ffa144b --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1de41ed818f5ddbfb2347de66e03fde6718016b7d0fc5624ce5094479cf7c580 +size 25920 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..4be3cf7a2b0758484e6ea5e98295209b55e59693 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0aba1b1c1e050e569330024c0e23a5aceeddcf940645c2837afea45c5e5d9c1c +size 13607 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e4e81391a491bc01f6f65b5c7c5fcf5b29ac59a --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01457e373c670f039963831d1ee63402d954df26a37877c20ddc5fdf314cc212 +size 13518 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6796eeb868ddae57e0b3344a6e88738405bc439 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:794e37c1bb334777c90e3be3652dff2fe0ed4f474a91ef05fb1bf5e6464cbbb9 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..39ec3efc1883e5fec9a908b670e50a2843f42f48 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0401de9b930ad8f35a4b0f18cb92cab4d5a1babfb65908695aef020a043f9ce +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/model.safetensors b/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1bd6ab5dff4bf6d037cbb5c42e0b9c6161f640a6 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22c960171ec1443b613aa31b7fbd06dd5d9beb273042cd17caa7949dbee21dd2 +size 18890032 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..780e4283a66ddc3e31f2176215178f7e7567434a --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1190e44c70e4111696a07ddd70ebae751f49e6aface053977732db333d51fb31 +size 25920 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..eda0c2e1552b9cdf132eca52e8c7a89de04de00f --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c44ea8de2666f79ad3e07a0244f5eb39d540c5daa7dd84d02fcad7bb218d8684 +size 13607 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..3570b95d4fb094a31323b979c5e4ed1cb85c8b26 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c848c87529bd60ef5ff5f84e18f2d57cd2340218f375dbb00fb850473170dfc5 +size 13518 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c443cf9624e12519b03e20a17090f0b989542a7 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dd1b2d5d6fbf1f0bff29ede4e4f3a6aec3e7ac3ac4a1d7fa1a15e4ef918aa49 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c597be7d080c150131c626291c0331919e3fb1b7 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:521918a28a679f55bdd70525ddb7f4511fef527afb54a3d89832e1cda6798fc0 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/model.safetensors b/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..56a12558fd28ea06da8fc15a47693a2c1c581447 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2aefa1e49c2a7eae944e788544963cb974eb84fdae6196bf7af674dc7b34a87 +size 18890032 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..605882289132a68b4d26f604f1de47c7d047d539 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae58f0a04b13e14b7c17553117e78be316745f17ac6d8578a380a12101ff06a2 +size 25920 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..00b06690639ab63b189bc179fe7cd9ed8417997a --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a88e035d78f9d5c6939446be3fe6277ac894b642b7dc24b07fa0876861441ed +size 13607 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5fae574e2b5ecf2dbaeee4087aeca770bca5ada8 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5773b9be7562c767190d75404ea11a8f3393fe509f30a1e66de7adc950f07a91 +size 13518 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9700fcbd6dc795534fdf32259e7a5e5507d1b9c --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cb480f58d59af4a0215bfa3bd77ab1ef496f674256eee24a8f8c4ba9967da33 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0012cfc31144f183294d7298179b8e396a573912 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a047a32b56d18f415c8995acc23db16663eb9ec4ccd929553fe66eef43d8b75 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/model.safetensors b/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..df7673a8ccdbded65b14e12b1332d34fadddb17b --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aebded1a5b8d00bf1e3f6a3481b1d239dd73768a3823b6fda1da54e7919e57a +size 18890032 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d200cbf6bb82e85ab106b0cb73f519b2bed55bdb --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:479494ee82efed5bd125e350b3a60aff69dea9e10df540617d04fdfb69eda96f +size 25920 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..702435040108a8d76df63243f73ffa3c96f4c426 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:235cc94c506f4307592780eefdbcdf34979fd6d562e0a32e3a0ebd57b05388f1 +size 13607 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5709313fbc87a1fcb2cbce019d4bb3c723ad72a2 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:837a359a56e56b06e4d710300bad42d78130da25e7c5d0c3304390c302e92946 +size 13518 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..354d6542ee200306ae8b5ee9febbfe3a79523513 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c71d32839b8d80384a5ddfd68a41b0a3d211fbe656dd8d56d3001edd818eed88 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2e5f8eb284bb4b21a85318c4073aaf5d1752df4 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:050773fa53aeed7952fc3a0f4c0122363ee60c3b9e5056c3f5ddb93ba9040c09 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/model.safetensors b/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1340eea14ac7d3015a023e9ee468fb692e9bf122 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e04822544c4548eed478db054ab025805f12d1101421247aad3a6ca7549628e +size 18890032 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e50b4ef559db8c9ed77c42bd0c5c12c24e248a95 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a80a6f7da2331e0c920c6961a95720a4544a06fef83228ffe01beb758ecc539 +size 25920 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f84b9fd7876af5fc6d4be1ea8d2178ccdd2579f --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d555134ab86be5bd65ca64eb2b5c73aded934c46d680671089122b3042e8d22c +size 13607 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6f09713ef70754409c2c13c2eac4b09f9b35737 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33253cb67d7a0deb5338c86920f319c89ccd9fe6ed905a1861cbed658a6c8cb2 +size 13518 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2a8f05f3c245bd28841138cc92d9bd4d00a8b6e --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92d522d34ecb63b4460b07a2bb559c38608d84b00283829501bee66be39d1796 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8a2130f721bc84cc4f5a3c1d409e946b47dff47 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aca56e636ab39565654ef5b0d5536e70ac0f387bdb58033e6d3d4aa65fb6d756 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/model.safetensors b/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..25ecf9f261846f9d210937bf9d74bc13594c1572 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:499b752da134a9618e0b3a9dfbe119757aa19609a1414cd3df5baebdabe722d0 +size 18890032 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..13a0cdcc17764d92f6e1381d91e394609efcc874 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ad955628d24478f57f57810757b1dcd9e498c52433ee45b71256cc4b238687e +size 25920 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb71dca4e4803f50fd14781bd16830a94ca77fdb --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:892ebbf5cccb307a4bf128ee394c62e744093398bc99c0da2567ca94d88a641c +size 13607 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..208081d3f7fc66bf7b2eccc31039ea605f2eca69 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c664288eb0271c934acca363030bf5e822e3dd403a56d441d9af2f661d232025 +size 13518 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..16d66ce6b098290dea613b749fd7cc737db9fdb0 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f52f97487eac9ed8cda3ff78194d6684a51dfa835bcb6c6c8cd744156796f7f6 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a87e48c8843de7ef69bbec417d17cb572a5f0904 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28a175795558e4ca08ce3293aca22f6aae432b3a2843b99376afc3fd7ae6dbbc +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/model.safetensors b/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..abc8c980cfa4e7bcd9a7da40b7fd5746c09cc211 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bdc242f26a3aeae153ee287d6b201fff47a957b3f6005998be703cee66b3449 +size 18890032 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fc87f65d7639e56a0c583ac485eb50514b645b9 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08255b0bd91fb7239e55ef6f20683f7e3f44ed59013177d2e2cff16a922aa7cd +size 25920 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..0fbdc9a4b3c3ab1e61cd0f6bf0c1cd370b0eb564 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:128d080e509b8201aad838902929d37b5dbea7ac91fd81d9cdca48f86908c74e +size 13607 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..474f0002ae9ef7de7e41308051f63778bf627282 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0aa35aa73d43580cead43bd54af20881c0e8b161dbc5c9fe68e2b5e260687c23 +size 13518 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1db967632efea6d37f57604ea110b82b20dc0aed --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d9921ea5f0ff47be599577be968c808a2cb6bf67cd466dc54dc0b7951f997c6 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..7571e43e54914bf35792af353838b2c69e1f705e --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9742d52f10b754f1985e72c65f7b995ef0e0934aff8290eb6b45e2a6fcd9258 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/model.safetensors b/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f36f4229ac5d67b10681ce18455b4338d69b20c3 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9ab051d658a6a6d084635a8ce935ba8e42836d42a4027e83e4ca36c1d9eb714 +size 18890032 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b584075a72dd201cc8147ce76ed0d09f691ce28 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce180e9a837807af0281df0eed9aa38e902afe62ac584220a2b0065ea5e07c4e +size 25920 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f799b6ae571adefd06009fae328b171b1b19592 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4828649bda9675aabb6c7214f4b34e7985f238b414e9a7e55144483363049f7c +size 13607 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c0463d919e91c4f1fe029f8c2c08c4fba810f71 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfc5b2f163e092bb94bf52e9ae9a0119543b23dfbf3d6e2c34904cadb9d8e1cf +size 13518 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8468148e917f23c85ec31bae26b5af6c745b4d2c --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8417d3c69625e62ed082fb306f6d2dcbc5aa5c721cc917e303804bc82e5d5236 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..681437bdfdcf6f33f65585731ca17224b2c0eefe --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a6fb629624d617e098b6d4f605f84223de7f6e397d291dcce7b8bba148e2d4d +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/model.safetensors b/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..79beef1d194c29782da0eb0c117d31201ece16e8 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5c1eeffaad74528c04cd03a3a5339848bb6cccb18f06d8c7c0704709192effb +size 18890032 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d92a642d90f90314701137a2b87f3d0ccef9269 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:715a368d6e41a813b4e5dae76e2487acd62a4a8dc0d92728656f079d3f4cd810 +size 25920 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..48b265ebdc332ca262e5618de35da660cc5bfa05 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c333ef9dcc19c05dc72de21b5bb88dfefab43d7ed17e921a0fdc61bc7d9f785a +size 13607 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..168826fd9e8b6508f25ae74862ffd81aac5fd299 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81fab2034c396defdb76c999cffdc6fa8cb260e1a3602f2bf97f8ac6253594c6 +size 13518 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..dba1769621275f5f5fbc5b21d4e35dfc4f8a2329 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d0d99a3606682b9d8986bb24d2cfe82343444cbcb2d7885f265feb6bd723fb1 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..911ca333003e2d956d528869cab242e6f23d2974 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d5b99d755e4e3beee075079c6ff2c6ddeb19576550c0806decb583af4850bf4 +size 13523 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/config.json b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..03af2764551fb0c668e41544170a64c51da927c1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/config.json @@ -0,0 +1,10 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "relu", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "class_type": "VanillaSAE", + "module_name": "fastsae.models.sae" +} diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/model.safetensors b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7442fe0c9d0be35619ae5b75f2b1b72936ba64db --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24fe40197e7c0ac023860429a81b35c27f128cc48161a971ea3ae58611158241 +size 37776696 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a028bc65a7f77010b4c848cc85e070643301a5f4 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc5932354357b593fdb3e3fae6d1d495798b4fc286f8ab43ec2e72cb6e622c8e +size 50496 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..f891738d36f1006825024cc2944acf653e62ded1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e764b378068f05ace48ed14f013ee9158659f5f4d807f50289891f94954e946 +size 25895 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..e113c59733a61a98d9d77ffb28e9035b6787fdaa --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 122.7903529296875, + "l0_per_token_0.001": 122.29861499023437, + "l0_per_token_0.01": 117.96412387695312, + "l0_per_token_0.05": 100.63427084960938, + "l0_per_token_0.1": 82.8563763671875, + "l0_per_token_0.5": 22.8890548828125, + "l0_per_token_1.0": 8.46724619140625, + "l1_per_token": 44.45847347420369, + "r2_score": 0.9101844750520658, + "l2_loss": 0.05979657077581726 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d3d71715d107137c0c3985de57cf5dc7c0009e8 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe263112b1835fe914d4e6f6e3fcfb9f97b9d6e919c5406b3429d8dc866a57f0 +size 25806 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..01822f189ed50c7c8e344a6f216bcdfc286c69fb --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e540b8130343fedee072a1b1eabbaa2b2e8871d895deecfb62e54ab334d67748 +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..44cf85e04e9e1b45a0d916b2d986f36d0a25e040 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7828cb2b4239989cd624aaf91c5f56276087f36a0d18b3a9d441bee88a8bfe78 +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/config.json b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2df92766c911a9cf15a36365f5f0d91bde86a2e6 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/config.json @@ -0,0 +1,10 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "relu", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "class_type": "VanillaSAE", + "module_name": "fastsae.models.sae" +} diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/model.safetensors b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..831d9232b5adceabea17f1e3767ad6840f5bbbd6 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b7becafc21932a859c8c7814e78af3e8712bb3649106061a2cdb72f5e6081fb +size 37776696 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f8362ea2ce8d034f2776caf2e0dac2f6fd31a2e --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa1a5261bc6ecdb59df4e727f704867f4dcee0d2fc1d53138e6aec0a23edceb0 +size 50496 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..461d06d47d8f9b6a25420101e3aaea7a08a51ca2 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67617c7bcc68a4c079f95836bdc2b69e641cc3481b04e361d6a96d5488beb3b8 +size 25895 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..fccc521702bdc743310c8488148e6e32bc878bd1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 122.8485640625, + "l0_per_token_0.001": 122.35689345703125, + "l0_per_token_0.01": 118.02145864257812, + "l0_per_token_0.05": 100.68524404296875, + "l0_per_token_0.1": 82.9023759765625, + "l0_per_token_0.5": 22.88810537109375, + "l0_per_token_1.0": 8.489296630859375, + "l1_per_token": 44.51770745628815, + "r2_score": 0.9100155486423547, + "l2_loss": 0.059909037078497644 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b311db536758cf2b7d1f848ee543f495bc393cab --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9332a0b38db270a15f22e64fc8cbcc63c12b277ad78d1e086bc3a1cb334ecc22 +size 25806 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..143274d531d81fa45b40657964a8595e2388ee4c --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29d6a09fcb98744bb10e27ec3aa0001cb60df12aa012a33631fc4c8969945f00 +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..57f4e60bc8b709fbafb0ee4e61ecf8e0622a427b --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:038c184eaed8358903219b4403ec6a5633917ddedbb74fa51d0892d9257ff2f1 +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/config.json b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6ff526fe86b1fee16310adf1b05c81222284af8c --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/config.json @@ -0,0 +1,10 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "relu", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "class_type": "VanillaSAE", + "module_name": "fastsae.models.sae" +} diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/model.safetensors b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3a17d1f77bc47e8557e42192ad2ea8e491da9a30 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41b89454ba075af3241ffc02750a5cfe66c08b74e55e3d560bdb237dc773a8b4 +size 37776696 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9021ddcaffe9288fb55267ff39c22ec016f9093d --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:810017ca81611848405ccc12a2b8175781f1508bd57b7cd4703ebfb12f842b8d +size 50496 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..1899553af251c677cc37b89a6320f2d0da27904a --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c19464da23013f78a009de703aa33766e6e99dbbee16109dd4ed3dbf7b816e7 +size 25895 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..d905237f88cc046462ab5aef43e42fc92cd6ce2d --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 122.10298837890625, + "l0_per_token_0.001": 121.61520112304687, + "l0_per_token_0.01": 117.31383422851563, + "l0_per_token_0.05": 100.11517709960937, + "l0_per_token_0.1": 82.4711703125, + "l0_per_token_0.5": 22.860000341796876, + "l0_per_token_1.0": 8.46036015625, + "l1_per_token": 44.22595241535068, + "r2_score": 0.9098951397383302, + "l2_loss": 0.05998920183348531 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..002437ceed6574bf7b67b0d4a4c1eab56bfec7f1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:123859e5bd29793a4845c24c2929c4c28228a49821cea79a95d524c54c5f92f3 +size 25806 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d833dd62553d5a8c099b6e1f02e5f51fa23cad3d --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e80c5ab2641e9f9e401707078934adb590bdd6e8bf5a2853e8ae6115166fbdc +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..901465d3fd051c8200faf5fd055eee7f2518e19e --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0663dc04019a2fac9289b9fa84d49965f86d2b85d960c42c3a2a46008d27a4db +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/config.json b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..03af2764551fb0c668e41544170a64c51da927c1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/config.json @@ -0,0 +1,10 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "relu", + "activation_name": "relu", + "device": "cpu", + "seed": 0, + "class_type": "VanillaSAE", + "module_name": "fastsae.models.sae" +} diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/model.safetensors b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5c8caec2e0d8d4cab30cef91f160e0171b447d96 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae6e868195e1b7b0b34600bd6614a0c0dc65c3f65573357a97b1286e098dadb1 +size 37776696 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e424593cef6bdc9e817032c4f0fb382cf2af970 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:275b72ca150d139761fd6b3582ce2d98b50bcf2c6c6bfd353d5facc5c5bdcea6 +size 50496 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..c58b28e54a8e1382aec58bdfce187b3ae06c5ca0 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19cc2114dbc9ea708bea65fffef3ab00a4e3231bac117408b91cf76489fae5a5 +size 25895 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..171ead5c4f3abd8b949bcaa370e64d3cc22449e9 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 34.74092470703125, + "l0_per_token_0.001": 34.67276591796875, + "l0_per_token_0.01": 34.0662400390625, + "l0_per_token_0.05": 31.525312890625, + "l0_per_token_0.1": 28.67179970703125, + "l0_per_token_0.5": 14.555092724609375, + "l0_per_token_1.0": 7.453949951171875, + "l1_per_token": 26.927988527501665, + "r2_score": 0.8597877585491807, + "l2_loss": 0.09334924251024752 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ea4ce3032f9d46aec1a36b370e8639ae87fe7ef --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ce466060d32d329803f2dbf60895b074706b5a065d23137ee1f97bcc4fb8d0 +size 25806 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d4b710e2788002450808e1762886b9caa055001 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da229d86ba9837070a4f4ba553b37e8d0b88e6a1250793f2b125b7a631920296 +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..089dad7d2341657d0aedbc9959288aa84bd09e2a --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5813b314125dc21482b724ad342abb6f8ae2cc6ec1351b0b85bd01685c04fde +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/config.json b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2df92766c911a9cf15a36365f5f0d91bde86a2e6 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/config.json @@ -0,0 +1,10 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "relu", + "activation_name": "relu", + "device": "cpu", + "seed": 1, + "class_type": "VanillaSAE", + "module_name": "fastsae.models.sae" +} diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/model.safetensors b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8427b387f7ca0d339799d1e53cebccc97bab1c41 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:276fe1883fa3905e319583e4da52c57ec29bff4f979498d858f58a9fac20c742 +size 37776696 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..26da3768ce474a3d964d49f1cc255c4f8c713177 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4935a8296ebefbc436daf102e42cf6d6702ecad37e3afc7a30b17342875477d3 +size 50496 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..52fe227cb1d1c51030edff825a51b33bdbfd4801 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b7bb06f493a725f767969873c410cd0fc52369d1eea3b399e65b143498858a2 +size 25895 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..64d4ab70ffec39b7efaa01e190fa9f279c60be28 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 34.2394486328125, + "l0_per_token_0.001": 34.1724388671875, + "l0_per_token_0.01": 33.57553349609375, + "l0_per_token_0.05": 31.07442255859375, + "l0_per_token_0.1": 28.26710791015625, + "l0_per_token_0.5": 14.390334375, + "l0_per_token_1.0": 7.40446318359375, + "l1_per_token": 26.720831672987465, + "r2_score": 0.8587411297408798, + "l2_loss": 0.09404605760594337 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6ae350f0b45839fb02c6f77773e0d5700e37aaa --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad64307cbe2b30035c1e8d265bb791b3f902d3cacb716be91b96e989feec4c7d +size 25806 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e563215fbf94bd59ca6608d2ec112e192ff1ef42 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76ede488b5b628752a78e697e3f8ffae004bbd2cbdec51331b556237602976f7 +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8feddb06ed7f0b948c714a916c6fb001dea5535 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c7b7bdaea9b07cfafd8272735152d8b72c34fa93bd57aa8401b2bf5958c9816 +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/config.json b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6ff526fe86b1fee16310adf1b05c81222284af8c --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/config.json @@ -0,0 +1,10 @@ +{ + "d_in": 768, + "d_sae_factor": 8, + "architecture": "relu", + "activation_name": "relu", + "device": "cpu", + "seed": 2, + "class_type": "VanillaSAE", + "module_name": "fastsae.models.sae" +} diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/model.safetensors b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1c3dc1726ab3c3caf43b0d6a518a4759344e107c --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ccf2131af7c462aaea63272f18aeb1d19d3d3a0bd23ca9606ff97213732bf08 +size 37776696 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a63efa7415ded0436425d09f6b38304635cc9d4a --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:711c7b878bb1a27c0458954e28c6c7da7725bef12560c97f3b4c96045a460865 +size 50496 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..77d4e329dc05ac7a583c8a4b9d667e15bfabfc7b --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26bc4125089f2d9a65fd6bb3867184a125103347570e9594d27e59987708af83 +size 25895 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..c83299ed60bf157d8f4b2763dcf4d53c69711d5e --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 34.3545751953125, + "l0_per_token_0.001": 34.28735971679688, + "l0_per_token_0.01": 33.68863881835937, + "l0_per_token_0.05": 31.179219873046875, + "l0_per_token_0.1": 28.363402978515627, + "l0_per_token_0.5": 14.430056982421876, + "l0_per_token_1.0": 7.418709375, + "l1_per_token": 26.68443019515859, + "r2_score": 0.8587780545584602, + "l2_loss": 0.09402147413366387 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e60d46f1befd04e870ed90faf11c71ce8b3f1b05 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:011b52766d5bc3ff96b4cdd64650e29a2898ee14c8041d14c66f1d8cf60358cf +size 25806 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..def4d7f899643c567e85b40d51e1ff9cb500eb04 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f649d462f1bcc7578eb304dc1ebfb7ee1eace4b9c03ca4af7bb8c2d208614be9 +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd236a6886b0a9f71d9d971e9c36c768bc2bb000 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6baadcdc59c84b12f8d1a131c98d764e52f7d4675eb6993b84a5713d64d0e73c +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/model.safetensors b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c9581cd26f6ed52cfbb01195e58b5509f69c0a69 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37e8c480f490b4981d36eea38ffe243c12f8ed56db5b784f5714e243272e8b2d +size 37776696 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a431cbedc2d7a16f55453c01764430aa575f5624 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:874ec4e4c15037e3e9901fe381f2a88d3447bb44f30f57277f69d0c8e57aa4e5 +size 50496 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..67d8ede58ae4f2828c0bb723e884068d2175e738 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b981382ed3232aa0acf7af757ac654e999f9d5e925805bf246b3b7a931c228a +size 25895 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json new file mode 100644 index 0000000000000000000000000000000000000000..054feb88f869af03352b0909a681fc80a7e274e3 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json @@ -0,0 +1,14 @@ +{ + "n_samples": 20000, + "n_positions": 20480000, + "l0_per_token_0": 61.33215146484375, + "l0_per_token_0.001": 61.16149399414063, + "l0_per_token_0.01": 59.6492275390625, + "l0_per_token_0.05": 53.43855620117188, + "l0_per_token_0.1": 46.728418701171876, + "l0_per_token_0.5": 18.497915576171874, + "l0_per_token_1.0": 8.085027001953126, + "l1_per_token": 33.40253760390999, + "r2_score": 0.8844004791704476, + "l2_loss": 0.0769628071866419 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ceaaa2fa57c983eaa649c799131f356d062bf89 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f3994a6657b86e755551eff246fde6d8572826d3b26e03e6d88b15577d04da8 +size 25806 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8376f091b8d4f6d88c02c20189ab58832d8fc320 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32b8b4c962a8d5987479189b9c8814d0f8199017d1e0e64d15e440cd00d50258 +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a84bc88f39308e93fc5caf5fca06b6700fe3bd83 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0def65e8fe4fe30305e2f3d3dc6262afe69fddfc7a5d9b12b6af2bf202912a33 +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/model.safetensors b/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3e94b7b628289ac505fe874631f7476b2090bbc6 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d770ca7ab7db7ab4ff694651e5c60c3ecaf493036becb2eecf5f171524bf5fe6 +size 37776696 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c2a2b6ab0149a2fb664714ee2a1d35f912ae045 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a11c6dbf3a5f65a61c29a53e8d9236e8387af03ffc32f176262335ed018503f +size 50496 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..bddbf9d2692098efdee2d19070401a007dbbba35 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1426f2d3683057c817d4084a6b869a63912aa7b103e96c7e869e3a155ef24e6b +size 25895 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..fca89c51283d3f50a60d68729eb9f27cb2c07285 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aba2cbd2f037a535fc1c789eab0bfa6c04a1301cedf3ad504ba1365a8d18aef4 +size 25806 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3a2f84031820b4d5f716b9ebe3ec86ac109a0fe --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec1eaef49421403cc813eabd8387c53a47c0bbed9b5c5c201dd72d6553cada5a +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb73b4e819c233268913d48df4e033c62ba5c79d --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb5140d87091cb79635676c0706f13606113c6a80b055de296d82684190d0073 +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/model.safetensors b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a246a22ec037187e382ac126185c44b08b268058 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28e5cb98074713a36096e2d40af674a1aa438fbb4c722d5b4e345cac1d75085d +size 37776696 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..4fe8bd69dec75f39ed7a79edddef36c8868f9bfb --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a70c1efc69ebe2a665d74512244d453e69fbeea5e2c7189d38ecfa4ba9d73b1b +size 50496 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..56dfb5c676af2bd5f656bbf9deea510a0446d646 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19ace87243f3a434039ff4d9d9072dd7b35127f31f910f401c4272e87a1630ce +size 25895 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..05923519490add70d86bd3fc94d6ebb1f66b5342 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c4286ab00d1ed681224484a69c369d5e9dd30e676e5e3b8a79e116cb1e88b94 +size 25806 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..70ab032e1bd11c5c60f268439108d6b1592ed798 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65ad5e1e2c80acbb077828a79952d7866305111408c89e83a07e4f5df21f1566 +size 25811 diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..cf48ecb23b5a78a1e272572fe8253b3a79bbf5d1 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json @@ -0,0 +1,26 @@ +{ + "dataset": "fineweb", + "split": "train", + "hf_dataset": "HuggingFaceFW/fineweb", + "hf_dataset_config": null, + "hf_dataset_text_field": "text", + "context_length": 1024, + "num_samples_requested": 20000, + "num_samples_actual": 20000, + "batch_seqs": 32, + "stats_top_k": 100, + "stream_seed": 0, + "sparsity_thresholds": [ + 0.001, + 0.01, + 0.05, + 0.1, + 0.5, + 1.0 + ], + "model_hf_name": "EleutherAI/pythia-160m-deduped", + "hook_target_name": "gpt_neox.layers.8", + "tokenizer_hf_name": "EleutherAI/pythia-160m-deduped", + "d_sae": 6144, + "d_in": 768 +} \ No newline at end of file diff --git a/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..09b48d3d3644d8563b724d228c78feda5154f3e7 --- /dev/null +++ b/lm/pythia-160m_layer8/relu/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:767eeeb9e922412601a3cf5759181e785dcbc967e07331de881a9c54c64b213d +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..15fc72308a47ed71de3917671ae6d86e0ecfa0d5 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1d0e143d9ffbaf873485e1df60ad7dbfd0d8c3ca3214bbd4fe8924032938705 +size 99648 diff --git a/lm/pythia-160m_layer8/topk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fe5478dd56a33840aa0a8675e8a97d754ecdaf3 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b4233f5da68b76e8b692901df31d209651f115f5a80d72fd08bda64d179e60b +size 50387 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/model.safetensors b/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c6deb9e8ab7e2825e0c5083f1140ade3c1e5bf51 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:297ced8ef628fed84a3108455a4ef46f89b46a498e85e9e0a7e7e935185b570d +size 18890032 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..25369b4d8167f4401220acd8088693c0808dd520 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f8664b39f84a7961b40b0e3ccd646d478422d3cc05283b1ec8476516297458 +size 25920 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..74b1c3a28e126cf4af5e210c8b3e3f3ca50089a6 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d0700470a3c870059b6a0d14582fc956caaf2522fd4db622bf10fae84610ee2 +size 13607 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..33ea8573ed140b397c94634a8e3622e38ca44360 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fb06f2c135cd0816b8c122df97cfa44d250bf2ff138abce1c7774c8ef0e8fb4 +size 13518 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c06c263a3917bd929656c856d66603c2cf2463ae --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5583a63d24a61f20d89705a76e5ad4e3fbba2b569cfc0e22d79105d330116cd6 +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d01a16ab6dc304f423bda7da765e933bd616520c --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07ef187a8de23c4c97301742b96d830df3e5d6154d70cc8867d2e4b1719d1f64 +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/model.safetensors b/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1ac97ce1bc78bef62414b4ad5ae70204bfb3525f --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:302d4b2198530e275deeff44edacf8572a1539ca33309532ce74ced1211208f0 +size 18890032 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..25369b4d8167f4401220acd8088693c0808dd520 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f8664b39f84a7961b40b0e3ccd646d478422d3cc05283b1ec8476516297458 +size 25920 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..5212edb3ff8319e684682033195cbfcafb0e7567 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c31c6dd94ae91aa20b9cc802bd33497c6d135308b4b2e0ed10064ea4c118f96 +size 13607 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8196081e39ae102a07511c7dfe8800deb66b1670 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e53e20e2d8b7df68383a2c193661197f1e73f377053e1413e167e3446dc91cf +size 13518 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa15afb1c62563789e391d4d8a4e9c6f452e1401 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f57b147f58032479f3a5ab09286e031725f932445b3c3bcba32ea9a97f791af +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed0d2e21d5c0de7fd199bddd0e7eb9855628e5b4 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f10b12ce248bd15ca290b090fb3eb165186916e96a369f51632d3a2965f82ae +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/model.safetensors b/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..02af4f5c2d313d3c3d9e175a195a8c4999fd7fdf --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb4c5309637e580603353b409d7e2cbd8a5ce1335a574bd9cf37f4f970a1d37c +size 18890032 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..25369b4d8167f4401220acd8088693c0808dd520 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f8664b39f84a7961b40b0e3ccd646d478422d3cc05283b1ec8476516297458 +size 25920 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..13f0ae5918771ba1e495f15f05b8e00b75cc70e4 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:363c7b1b7fc8467f3440da78113b5d971ca2418d8a3aa3107065b44df47a363d +size 13607 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d09f852d3cf3892e3a951329034fc157541e6ed --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70dbd00870de7958f0812f846aeff40d38e62ce6f5079d7653b290d83862a7b5 +size 13518 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..671fb1e796fafa3ff7a63c2e05de782799c55028 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4003645092d03b6b3ae0351aefb7f387f68dfc4637dff0aa4343ba0f7e942c9 +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..52535ec26ae1aa0d791fac147e31d8d9f72c183b --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6939f28eeda39424fe8ba12a9fbcf485032719962b2d83e6953219cfe0307a42 +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/model.safetensors b/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6dedf32d2575f87ff9f49faf81d70ce9b2343fe2 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3487398ced4cf634155550abc0632cee260ed69ddb13e220a4462b4ca406959d +size 18890032 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..22476d226c364cbdda1c2808c14b7ca570e296dc --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8cf66876f6b155c9ad52c0309e49a57494f55abcd7294eb01e0be14019e09c +size 25920 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..c768304bc5c50d7a01ab096349eb8ab7b30cebe2 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd145363c5c92acfd80bef71478083adaa8041f9274eb4e2efb56c9345dd858e +size 13607 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad1bf5d3a25c5ce54e2be32a55c7124743af95bb --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8c8ea8c5e189d4ab19403be2902ab62158a5ceac56f597449811afc58315b6c +size 13518 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..035053d9be35bbcfebe1dbc111a29663810ddc7b --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:756426b3c55a9ab7e80d1c0af10f2ac811802966fac978a9dd713d73dce7056e +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..2578619f86158f46cc8bd351caf904fbe4fa5188 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a299020665442791bef45a3e4bf7a1333667961a328825664f98cbfe467cd22 +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/model.safetensors b/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c6aed3c6c4fa8ea258d27abb95fbf86c1d2f1165 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1437896664b652432a3817e58090c59edb1e820017690eb45bbe72bf7ae49fc +size 18890032 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..22476d226c364cbdda1c2808c14b7ca570e296dc --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8cf66876f6b155c9ad52c0309e49a57494f55abcd7294eb01e0be14019e09c +size 25920 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..06d808020c26314b29fff5ec7dfe29c1074741fd --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e07029ba3b162e28592a82a2c3dcf1d183cbe898771f70f6e0613face8ccf872 +size 13607 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..45363613f97f60c2268bb4508e9c951e45144920 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89c2711c9884ab88e83593c9958005b081d98613e60ed448526d186c2cded025 +size 13518 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0feaa7c514023aee79f6c69a4b672e74983e6151 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdac2850d281701016ddf59706ff113bd0b45abe3b09180f916ef6b6dc543b58 +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b00d8154a3a73e478da0d15a90cb2118acc15c9f --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5f3f3461560653075e4bee23d4a85fa83a03f4f73396215698062f9ef4a5331 +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/model.safetensors b/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a23c616aa1ac75b0d93a9ee18bb9d8124976a7e7 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c95dce14fc6900a3c35458aa8921a5e0339c8e1e144fd20c1c139c9f135c4fb1 +size 18890032 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..22476d226c364cbdda1c2808c14b7ca570e296dc --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8cf66876f6b155c9ad52c0309e49a57494f55abcd7294eb01e0be14019e09c +size 25920 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf701bef5f6c33d4539b4b900c217f6f518de5da --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51996626ba7d6359104bd02769342e402345973152c35a89b6a6841934c5b3bc +size 13607 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..95b48907e429cba5f845da606813d2696e51221b --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1910cf1cc8b97c37d2a9192189e2b201a20224a1d3c107c25feef9f5e1d85c5c +size 13518 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa0ca8edef2eec3ef9ea5e900f96e688756cf07c --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56a4c25bbe804fcc76eaea8fd65993bd43e5279e4a4391f7d020837d34c861be +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0afde9fd8cc3b03e427062298d2c96a69929b84 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d994d949001672166f0e5c5bab91987ce8a7961057dc85974d493ed21add058b +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/model.safetensors b/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d4b9dd893474835b656590839d49b9ceaf34ccbe --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8c9a44b23471b245321eaec06f83ab60ea72864ac01be0b742fc9562d1a932b +size 18890032 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..05342aec6090b6546064ba3d08efbd5678ed2326 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d32f23fd81286996eebbb066afd1b434695b0b36f36a328d7242c2e286a4e12 +size 25920 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..52aeab52a9502a1bad2f67cdae7220a5f5b2759d --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f3673861e2fcf213a8c8d1aae12328d767fe6745a03cef9a929cdad297832f6 +size 13607 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..17a258caf644bde71f310410aa7ab872dc5a7940 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bab0c41d14c2dc28e5488eb66e1946a52031be6df291f72731bb43103736cae +size 13518 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7659e9dfe99a813fbb50e3a2cc595d36e54cd10 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c38b62e18603790792ed5f8a185a38ca5af0e26319455326c1f017df2b325632 +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..09beea8799119c4b5a831a1ea463c6254f0a321e --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:326767618ec1eedb4b9a1ac54d56de4504088c5393c5cb910ec9ae20c6a2ffd1 +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/model.safetensors b/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6e2b1b3aea5cff454ae17f012a040f879c3d2952 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a5272bcdabdeaf6831a4aff042d7da85a146da77feddd561529b9e04bd90eeb +size 18890032 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..05342aec6090b6546064ba3d08efbd5678ed2326 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d32f23fd81286996eebbb066afd1b434695b0b36f36a328d7242c2e286a4e12 +size 25920 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1f474d44f092c3208cd075d812fca86f7c49828 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8a3b34e3d867914997a89746215a5c2c22ffafab82dcc0d95d15658555fd2b6 +size 13607 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a29631a5c850b0f42564cba2da7c72375bd8ca7 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1405a7896a5aa4d2028d5da64042a7ffad7d99bf86c2673dca5e39c581b3606 +size 13518 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b1c70be260116bade91c30a44451af3b2668056d --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27435276ca00e76d6e5135f10e499dea3aa5245a02fe8f22e577ee5de3e7d8a7 +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..98123b2c271d4c55c20307cc55428eabd64e5225 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:331b3248aed6d1de6f8dae5a7cee217987de3d22cf7e50d6d46b16a75275b882 +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/model.safetensors b/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..087336ab1ac793cd90475f5c73e9cddff2dddef7 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed758f3c0d6f366a91af624cb2d3ef8bd983b6b551747526f01f970668c22576 +size 18890032 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..05342aec6090b6546064ba3d08efbd5678ed2326 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d32f23fd81286996eebbb066afd1b434695b0b36f36a328d7242c2e286a4e12 +size 25920 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..dac9ac45684533d7ba06690cf8edf6cc2d1acd6e --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ce5f33c2da92676905331a82a68aeff515da0bd6cb66586cfa8503eaa66c01b +size 13607 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1239f1cb214e9f67666bb11d4556c10bad0962bd --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04ec80f08231f42c8d54e133daa832529df7f5882a97106b4dd8cc6b8ef94d69 +size 13518 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..69b00f56df047be7bca32e36049287029e4405e8 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2f62709bee62467090d5fc82624562741526ea87bd0aa3425219f2009db0129 +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0644c03aae486183000f84a5d063ec64713acdca --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x4/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d5b704fe9c8c745e1c280b430b451834b2d2e4d5985d065332ed93f00d8ced2 +size 13523 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/model.safetensors b/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bba9a08f85c53f8a07950906476b489b3abfee13 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fed78ca4850bc1e940b3c1e12384f0f360e7b7746f82ba5c9695505d25a65e42 +size 37776696 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..da8bb52901fc7e8720225d22c98d91c5121ddac6 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62cb2841c9a88a1c8f93b83e30c4831c7f45af5fff1fed059fc426b953645adf +size 50496 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..45603ef946c5c92a93123e897e451973b12d8b6a --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:361ddcfd2c8d1820ddfc3a8c266dc66592e794b6d4927e4be57f6c44d057c02e +size 25895 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e5521f6a386fc8fbdf87225c685b0a405ae9d4f1 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48087dd3612fc164207340c9ebd833bd93548b712f0f7a04dcdc1c19a30be26b +size 25806 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..eda233d227d8a8d82e067b2dc0bec096d31fbb74 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe997add7ee7dccd010731b4f3e6250bee0665f0ee642fe18199fdb7835ac295 +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..1bd31eb41de728bcd8a4989bb5724cdb83514165 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2893aa2ece1a1d901a3faf47b54832aa54304fb985de09633f8b971b89b9619f +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/model.safetensors b/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7db4260ea694983563ff1ed81d54021851b7bd90 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1210c18552812b767f61062a72d5e83a3ce3c8d4c9615b994dc7a0464b97c526 +size 37776696 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..da8bb52901fc7e8720225d22c98d91c5121ddac6 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62cb2841c9a88a1c8f93b83e30c4831c7f45af5fff1fed059fc426b953645adf +size 50496 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..6306610e0e1269e6688f635893ba873b9b2e827d --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5248ea00ba02c062165212a0c4d0f7d5bfe15a8f954d3e71ed6710bf84edf00d +size 25895 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..706dc09295a9c38b45cf0059b56b3b6086a57f77 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67f055293f027af6f8541ba4fa7635eab3e7045a1ae4cecc33b6cc18fb10e1d6 +size 25806 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c43be9d868cc2e763603921f63d06b9e478e20b0 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c03584a18bfba205652584e8f3af2e1f06ced8b1c7f9720df8b6caecd4699cd6 +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a775b4298e1bfa4bbc308a49b97a396c1a9708ab --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa35fe9036155493a623508baaa3a4bafb77377e67dac7a95cb600e39f43cf8b +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/model.safetensors b/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..41b87818339e4e585a208a6da3be47d502506e02 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c575728a4f1cb4fb8a48a9f6956e2552b2da8a0216dbd157f2e4c405e0f1827 +size 37776696 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..da8bb52901fc7e8720225d22c98d91c5121ddac6 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62cb2841c9a88a1c8f93b83e30c4831c7f45af5fff1fed059fc426b953645adf +size 50496 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..a102947464288ee7cf42db5cc3bd0134a148a284 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34b13248c59c5c6d352fe5479981a43f5dcd5403659ff7b4faaefe852618172a +size 25895 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..25d819f001564867e15fb29ec193a982162c7afe --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f761864b58001be9beffb8b4e2b8685379dac3dd340ce972676a6026a82db775 +size 25806 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ecccc7592d773a92e2e405aeb371c44b192a5e8f --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d57c048c831cfd0c0a3484097b8ede5af5482d87240c70dd8e1d2b007d35fe9 +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab3f0db051772cf426086f8ac50e5df3a29669d0 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_128/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3abaa6738c63065455abb835a0db8567aeea2c52f1033a1576157e85e61a9b35 +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/model.safetensors b/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0ce90401e3d1490e61551d34d75e480f1e0baa1d --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96572413921c5a9b2a5b96f30614497606176752bffab70fcdfb18be6cf179f5 +size 37776696 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d846a9c18af168b9ab640277af5d8fd537dd811 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd44067a3b156c4c657c2ac2d0d08d2df65ac472c009e7ae3a381d80cb26c91 +size 50496 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..fdfc68b93e843a457a8d75c2210d0f33844230ff --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab9b7effb0330ec55e919e26366f8928aa3634d36d1d40f6b6e38417eb327c7d +size 25895 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..f13e4c319f71675b6193807019cf4fcf91dc085e --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:703c5dba6aa32dcba60815b4404dfdeee87ae3a34dfc413b1531871e6d705ad5 +size 25806 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c99e0e79b3902021fe8c4b8df3b6481fd99f8ab4 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f8dfb43109778860bf4dce569d3b440637f8d9c7276614156cf583391922378 +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8448393032a1879128cba58e052eccde727bbb2 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a19308b74d86463c80084c38eaafe0fc63eade256cf4da8e8148f515e43ed2a +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/model.safetensors b/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..58ce31057cd22a36781cc97a82aa68f7c6d13f5d --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43ac97e7bf1e05b397bbc4bbc85955f73998373a7dc43dac5a1e1dba840233f6 +size 37776696 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d846a9c18af168b9ab640277af5d8fd537dd811 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd44067a3b156c4c657c2ac2d0d08d2df65ac472c009e7ae3a381d80cb26c91 +size 50496 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..3683deff498424967f94a8e4f149a3a9e4abd4a7 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32f5d37fe2cf4982d2b04182f24854455824c42e38c0716e750ab262b1891707 +size 25895 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..feba23e139e889b36fc423677e3f4c95354fb864 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be1544d609395adbd195fb506c279a0fe0356e2b9519413f3845f9e5835cc63e +size 25806 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7a859ff46a0ac03907a61e13661f0e13fb290f0 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b87554dfcc2e6b02df068be71872da6674b47d96be7fc65fb7496f43e2b6eff7 +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6350dfbd9ef30311cbaabb4f62995d84a42fee49 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:806641e2b5ddd99ba366b930ed54d215a16f2e89e5b1c63f1a91c4b457f074ba +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/model.safetensors b/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0438be5bdc00da754e813ad7a0e2d7ddbb09677a --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2d1d38a92cb080a47ea468e9ffe0fa01a6eb1a3fac0eae19cfa72e9f7afdb62 +size 37776696 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d846a9c18af168b9ab640277af5d8fd537dd811 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd44067a3b156c4c657c2ac2d0d08d2df65ac472c009e7ae3a381d80cb26c91 +size 50496 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..25c5f208db64f296aa77251c734c725a6a062234 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45c140d8d02f178e5233f3e72e944b1a0616a64e5d03e78ccd24b4cc862a1faf +size 25895 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..6bc8728a258c57a57abbd58965ec1ebcfe7eaaba --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9048ef2037a0924e4b57e6f2ac6f5e53508d462ed00d91e90b97d18b03a4355a +size 25806 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8f4701640e9045d32e8c1e03b4b220033278e40 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9cdbc95ab12d252af2d4a531342a10bda9aab43c13362a3df386b8f3f727e89 +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..88dd60b7a8930b7e7586b1766078cbfaff8baeed --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_32/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c7a319fe21240a12adc1c3dc6a5eace1aa521f27d39c606a959c64710743566 +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/model.safetensors b/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1c1e0c301343c0b9b0e17267ada72401d30d33b0 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:282fcf2800310d46c2a05694486d7efec074c2dc1e6d96159e87bbe54b28800a +size 37776696 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..abcd360e514b4464f4c5e0612bc68c71a618868d --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f384933d03e973cacb959093fb9e9bb2cf7197ece15e4b731d8942192fa0463 +size 50496 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..4021ca1f5ff35a66de86f3fe967a7fc75a48f498 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e63c6bc5ac8d7c78713ccfa4c4d6d96612195a1b45229f01381b8cf6b26da92 +size 25895 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..51e57b06a4b6d8d40ba2308076c214bd8c7fe03d --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e16d6d7eeaa6f24d5654b816302c9bb4916c40084ee1aa4faa0f461ca6039d4 +size 25806 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..b1be320fa23afcbf9a3a62df6d6a609d4032090f --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e56a4a93325bc26d791f33f7ab8f3d9755d87350c2e4ac5e4b06f35b8525a5 +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5bb45338530d56c8f57736649b763adbbbfad5d --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed0/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ff5551a1a2e5a5dfe6a5882a75c7abff3c6b92b79b337e35ee44c92f6a9c8a7 +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/model.safetensors b/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..971ea4553d9b6a498909324e0af36ee57ce43067 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0d3a57635f883e81f95ca41e3825af2a7db38fb7093c7f26995dbda9921a1a9 +size 37776696 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..abcd360e514b4464f4c5e0612bc68c71a618868d --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f384933d03e973cacb959093fb9e9bb2cf7197ece15e4b731d8942192fa0463 +size 50496 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b912ce00fee483a9bd48b40e1a38e43d122054f --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdf606bd8417e9bcebe9ec596c2f7acab972013366ceef59e1b3c88c88291a64 +size 25895 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c5a3dad01e40f20eb5fa6cac4d99a3c5c0de02f --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94b53471551b664e9cbc36ba36e0aa501002b13bfd1d9fb88bfd381b260ff523 +size 25806 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a381b04438566f4409505f59b67706f43d36d84 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37862994fa49f370b36456a8e64cacc156cd270998295418bb96c441d42809ce +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..0011b79104ce74c3fe6c9a411bb96ce4fc4b7280 --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed1/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55b562ed85435df646f75e74d006daa6293f6bb160bec7748bc08fedfbd05a13 +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/model.safetensors b/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b927a60fcf595b5f1838b841f5bc523ea809fc9b --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84d4c6a0595693acdf62d3922d472fe573317e3022c369ab20990630671cb91d +size 37776696 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..abcd360e514b4464f4c5e0612bc68c71a618868d --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/active_count_hist_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f384933d03e973cacb959093fb9e9bb2cf7197ece15e4b731d8942192fa0463 +size 50496 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb9d0f80f0a1e096579ea2fb8cd1a049faae3d0e --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/firing_rate_per_latent.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:761079e5e185c91bf81d246a4c6b90394570889d177af8d08b6476d12570eaa1 +size 25895 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..c642e5eb47702af3ce3a7b30da25848c179b670c --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/max_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21bc288ab1a5b164ee4c0606659e16c4823106b0d3034f09a357ff94084dbdfe +size 25806 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9cf9aad672e4d1df1886fbb8844e251f58e5c2c --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/mean_act_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a6cdf31ca147f909bee19195ba11e861b9551dfc50538dace8ba95a605f2f87 +size 25811 diff --git a/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt b/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt new file mode 100644 index 0000000000000000000000000000000000000000..37f6f680b3391ec463606f8859e7aeefaaae601a --- /dev/null +++ b/lm/pythia-160m_layer8/topk/x8/l0_64/seed2/stats.fineweb.split_train.seed_no/sparsity_per_token.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54a00cacb7f4b14d3b19177598bc88661ed9018d0b004da75f7583fc009c2c19 +size 25811