Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- lm/gpt2-small_layer8/consdl/x16/l0_128/code_metrics_step_final.json +136 -0
- lm/gpt2-small_layer8/consdl/x16/l0_128/dict_metrics_step_final.json +550 -0
- lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/config.json +24 -0
- lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json +14 -0
- lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json +26 -0
- lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/config.json +24 -0
- lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json +14 -0
- lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json +26 -0
- lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/config.json +24 -0
- lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json +14 -0
- lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json +26 -0
- lm/gpt2-small_layer8/consdl/x16/l0_32/code_metrics_step_final.json +136 -0
- lm/gpt2-small_layer8/consdl/x16/l0_32/dict_metrics_step_final.json +550 -0
- lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/config.json +24 -0
- lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json +14 -0
- lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json +26 -0
- lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/config.json +24 -0
- lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json +14 -0
- lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json +26 -0
- lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/config.json +24 -0
- lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json +14 -0
- lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json +26 -0
- lm/gpt2-small_layer8/consdl/x16/l0_64/code_metrics_step_final.json +136 -0
- lm/gpt2-small_layer8/consdl/x16/l0_64/dict_metrics_step_final.json +550 -0
- lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/config.json +24 -0
- lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json +14 -0
- lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json +26 -0
- lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/config.json +24 -0
- lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json +14 -0
- lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json +26 -0
- lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/config.json +24 -0
- lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json +14 -0
- lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json +26 -0
- lm/gpt2-small_layer8/consdl/x4/l0_128/code_metrics_step_final.json +136 -0
- lm/gpt2-small_layer8/consdl/x4/l0_128/dict_metrics_step_final.json +550 -0
- lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/config.json +24 -0
- lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json +14 -0
- lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json +26 -0
- lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/config.json +24 -0
- lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json +14 -0
- lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json +26 -0
- lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/config.json +24 -0
- lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json +14 -0
- lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json +26 -0
- lm/gpt2-small_layer8/consdl/x4/l0_32/code_metrics_step_final.json +136 -0
- lm/gpt2-small_layer8/consdl/x4/l0_32/dict_metrics_step_final.json +595 -0
- lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/config.json +24 -0
- lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json +14 -0
- lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json +26 -0
- lm/gpt2-small_layer8/consdl/x4/l0_32/seed1/config.json +24 -0
lm/gpt2-small_layer8/consdl/x16/l0_128/code_metrics_step_final.json
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": 1,
|
| 3 |
+
"step": "final",
|
| 4 |
+
"n_tokens": 600000,
|
| 5 |
+
"thresholds": [
|
| 6 |
+
0.5,
|
| 7 |
+
0.6,
|
| 8 |
+
0.7,
|
| 9 |
+
0.8,
|
| 10 |
+
0.9
|
| 11 |
+
],
|
| 12 |
+
"seeds": [
|
| 13 |
+
"seed0",
|
| 14 |
+
"seed1",
|
| 15 |
+
"seed2"
|
| 16 |
+
],
|
| 17 |
+
"per_seed": {
|
| 18 |
+
"seed0": {
|
| 19 |
+
"l0": 126.42603333333334,
|
| 20 |
+
"recon_r2": 0.943795649182259,
|
| 21 |
+
"l1_per_token": 289.4212340388216
|
| 22 |
+
},
|
| 23 |
+
"seed1": {
|
| 24 |
+
"l0": 125.811965,
|
| 25 |
+
"recon_r2": 0.9438612213988956,
|
| 26 |
+
"l1_per_token": 289.53429910061254
|
| 27 |
+
},
|
| 28 |
+
"seed2": {
|
| 29 |
+
"l0": 126.05112333333334,
|
| 30 |
+
"recon_r2": 0.9438580555624618,
|
| 31 |
+
"l1_per_token": 289.6181956334526
|
| 32 |
+
}
|
| 33 |
+
},
|
| 34 |
+
"per_pair": {
|
| 35 |
+
"seed0\u2194seed1": {
|
| 36 |
+
"alive_source": "stats",
|
| 37 |
+
"mcc_signed": 0.7859704474411874,
|
| 38 |
+
"mcc_abs": 0.7859711613635518,
|
| 39 |
+
"act_iou": 0.45132704906101173,
|
| 40 |
+
"act_iou_n_pairs": 12288,
|
| 41 |
+
"input_r2": 0.9913803291545396,
|
| 42 |
+
"matched_mcc_signed": {
|
| 43 |
+
"0.6": 0.8342782215099661,
|
| 44 |
+
"0.7": 0.8634063066238307,
|
| 45 |
+
"0.8": 0.8990084994065084,
|
| 46 |
+
"0.9": 0.9396039340469318
|
| 47 |
+
},
|
| 48 |
+
"matched_mcc_abs": {
|
| 49 |
+
"0.6": 0.8335566129871208,
|
| 50 |
+
"0.7": 0.8630353633737552,
|
| 51 |
+
"0.8": 0.8988650385049328,
|
| 52 |
+
"0.9": 0.9395716929205603
|
| 53 |
+
}
|
| 54 |
+
},
|
| 55 |
+
"seed0\u2194seed2": {
|
| 56 |
+
"alive_source": "stats",
|
| 57 |
+
"mcc_signed": 0.7828689400311742,
|
| 58 |
+
"mcc_abs": 0.7828697764077752,
|
| 59 |
+
"act_iou": 0.4478816509665113,
|
| 60 |
+
"act_iou_n_pairs": 12288,
|
| 61 |
+
"input_r2": 0.991334920205118,
|
| 62 |
+
"matched_mcc_signed": {
|
| 63 |
+
"0.6": 0.8309814935810688,
|
| 64 |
+
"0.7": 0.8613655575710586,
|
| 65 |
+
"0.8": 0.8984125478906685,
|
| 66 |
+
"0.9": 0.9399642265869411
|
| 67 |
+
},
|
| 68 |
+
"matched_mcc_abs": {
|
| 69 |
+
"0.6": 0.8310063654716346,
|
| 70 |
+
"0.7": 0.8610575981792729,
|
| 71 |
+
"0.8": 0.8984265823137941,
|
| 72 |
+
"0.9": 0.9399718561318022
|
| 73 |
+
}
|
| 74 |
+
},
|
| 75 |
+
"seed1\u2194seed2": {
|
| 76 |
+
"alive_source": "stats",
|
| 77 |
+
"mcc_signed": 0.783658469940665,
|
| 78 |
+
"mcc_abs": 0.7836587034705734,
|
| 79 |
+
"act_iou": 0.4499608370893986,
|
| 80 |
+
"act_iou_n_pairs": 12288,
|
| 81 |
+
"input_r2": 0.9913863229725707,
|
| 82 |
+
"matched_mcc_signed": {
|
| 83 |
+
"0.6": 0.8351966666901895,
|
| 84 |
+
"0.7": 0.8621449988701696,
|
| 85 |
+
"0.8": 0.8981497940397523,
|
| 86 |
+
"0.9": 0.939461465954832
|
| 87 |
+
},
|
| 88 |
+
"matched_mcc_abs": {
|
| 89 |
+
"0.6": 0.8341974655594866,
|
| 90 |
+
"0.7": 0.8619405907398374,
|
| 91 |
+
"0.8": 0.8981406120242229,
|
| 92 |
+
"0.9": 0.939461465954832
|
| 93 |
+
}
|
| 94 |
+
}
|
| 95 |
+
},
|
| 96 |
+
"summary": {
|
| 97 |
+
"n_seeds": 3,
|
| 98 |
+
"n_pairs": 3,
|
| 99 |
+
"report_band": [
|
| 100 |
+
0.6,
|
| 101 |
+
0.7,
|
| 102 |
+
0.8,
|
| 103 |
+
0.9
|
| 104 |
+
],
|
| 105 |
+
"l0_mean": 126.09637388888889,
|
| 106 |
+
"l0_std": 0.30952494097287203,
|
| 107 |
+
"recon_r2_mean": 0.9438383087145388,
|
| 108 |
+
"recon_r2_std": 3.6978134090712164e-05,
|
| 109 |
+
"l1_per_token_mean": 289.5245762576289,
|
| 110 |
+
"l1_per_token_std": 0.09884011177990036,
|
| 111 |
+
"input_r2_mean": 0.9913671907774094,
|
| 112 |
+
"input_r2_std": 2.810736274817572e-05,
|
| 113 |
+
"mcc_signed_mean": 0.7841659524710088,
|
| 114 |
+
"mcc_signed_std": 0.0016118284469957588,
|
| 115 |
+
"mcc_abs_mean": 0.7841665470806335,
|
| 116 |
+
"mcc_abs_std": 0.00161185483533366,
|
| 117 |
+
"matched_mcc_signed_per_threshold": {
|
| 118 |
+
"0.6": 0.8334854605937414,
|
| 119 |
+
"0.7": 0.8623056210216863,
|
| 120 |
+
"0.8": 0.8985236137789764,
|
| 121 |
+
"0.9": 0.939676542196235
|
| 122 |
+
},
|
| 123 |
+
"matched_mcc_signed_band": 0.8834978093976598,
|
| 124 |
+
"matched_mcc_abs_per_threshold": {
|
| 125 |
+
"0.6": 0.8329201480060807,
|
| 126 |
+
"0.7": 0.8620111840976218,
|
| 127 |
+
"0.8": 0.89847741094765,
|
| 128 |
+
"0.9": 0.9396683383357316
|
| 129 |
+
},
|
| 130 |
+
"matched_mcc_abs_band": 0.883269270346771,
|
| 131 |
+
"act_iou_mean": 0.4497231790389739,
|
| 132 |
+
"act_iou_std": 0.0017349504370717912,
|
| 133 |
+
"act_iou_n_pairs": 12288.0,
|
| 134 |
+
"alive_source": "stats"
|
| 135 |
+
}
|
| 136 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_128/dict_metrics_step_final.json
ADDED
|
@@ -0,0 +1,550 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": 2,
|
| 3 |
+
"step": "final",
|
| 4 |
+
"linkage": "complete",
|
| 5 |
+
"thresholds": [
|
| 6 |
+
0.5,
|
| 7 |
+
0.6,
|
| 8 |
+
0.7,
|
| 9 |
+
0.8,
|
| 10 |
+
0.9
|
| 11 |
+
],
|
| 12 |
+
"seeds": [
|
| 13 |
+
"seed0",
|
| 14 |
+
"seed1",
|
| 15 |
+
"seed2"
|
| 16 |
+
],
|
| 17 |
+
"per_seed": {
|
| 18 |
+
"seed0": {
|
| 19 |
+
"d_sae": 12288,
|
| 20 |
+
"d_in": 768,
|
| 21 |
+
"l0": 126,
|
| 22 |
+
"l0_raw": 126.4695044921875,
|
| 23 |
+
"recon_r2": 0.9436933468059865,
|
| 24 |
+
"l1_per_token": 289.3836344397387,
|
| 25 |
+
"stats_source": "stats",
|
| 26 |
+
"n_dead": 0,
|
| 27 |
+
"dead_rate": 0.0,
|
| 28 |
+
"alive_source": "stats",
|
| 29 |
+
"n_dense": 19,
|
| 30 |
+
"dense_rate": 0.0015462239583333333,
|
| 31 |
+
"dense_threshold": 0.1,
|
| 32 |
+
"n_dup_signed": {
|
| 33 |
+
"0.5": 3965,
|
| 34 |
+
"0.6": 19,
|
| 35 |
+
"0.7": 3,
|
| 36 |
+
"0.8": 0,
|
| 37 |
+
"0.9": 0
|
| 38 |
+
},
|
| 39 |
+
"n_dup_clusters_signed": {
|
| 40 |
+
"0.5": 1804,
|
| 41 |
+
"0.6": 7,
|
| 42 |
+
"0.7": 1,
|
| 43 |
+
"0.8": 0,
|
| 44 |
+
"0.9": 0
|
| 45 |
+
},
|
| 46 |
+
"largest_dup_signed": {
|
| 47 |
+
"0.5": 8,
|
| 48 |
+
"0.6": 4,
|
| 49 |
+
"0.7": 3,
|
| 50 |
+
"0.8": 1,
|
| 51 |
+
"0.9": 1
|
| 52 |
+
},
|
| 53 |
+
"babel_mu1_signed": 0.4519537389278412,
|
| 54 |
+
"n_dup_abs": {
|
| 55 |
+
"0.5": 4019,
|
| 56 |
+
"0.6": 117,
|
| 57 |
+
"0.7": 19,
|
| 58 |
+
"0.8": 0,
|
| 59 |
+
"0.9": 0
|
| 60 |
+
},
|
| 61 |
+
"n_dup_clusters_abs": {
|
| 62 |
+
"0.5": 1831,
|
| 63 |
+
"0.6": 56,
|
| 64 |
+
"0.7": 9,
|
| 65 |
+
"0.8": 0,
|
| 66 |
+
"0.9": 0
|
| 67 |
+
},
|
| 68 |
+
"largest_dup_abs": {
|
| 69 |
+
"0.5": 8,
|
| 70 |
+
"0.6": 4,
|
| 71 |
+
"0.7": 3,
|
| 72 |
+
"0.8": 1,
|
| 73 |
+
"0.9": 1
|
| 74 |
+
},
|
| 75 |
+
"babel_mu1_abs": 0.45511364936828613
|
| 76 |
+
},
|
| 77 |
+
"seed1": {
|
| 78 |
+
"d_sae": 12288,
|
| 79 |
+
"d_in": 768,
|
| 80 |
+
"l0": 126,
|
| 81 |
+
"l0_raw": 125.89824155273438,
|
| 82 |
+
"recon_r2": 0.9437663728355269,
|
| 83 |
+
"l1_per_token": 289.51439305719344,
|
| 84 |
+
"stats_source": "stats",
|
| 85 |
+
"n_dead": 0,
|
| 86 |
+
"dead_rate": 0.0,
|
| 87 |
+
"alive_source": "stats",
|
| 88 |
+
"n_dense": 19,
|
| 89 |
+
"dense_rate": 0.0015462239583333333,
|
| 90 |
+
"dense_threshold": 0.1,
|
| 91 |
+
"n_dup_signed": {
|
| 92 |
+
"0.5": 3819,
|
| 93 |
+
"0.6": 14,
|
| 94 |
+
"0.7": 2,
|
| 95 |
+
"0.8": 0,
|
| 96 |
+
"0.9": 0
|
| 97 |
+
},
|
| 98 |
+
"n_dup_clusters_signed": {
|
| 99 |
+
"0.5": 1739,
|
| 100 |
+
"0.6": 4,
|
| 101 |
+
"0.7": 1,
|
| 102 |
+
"0.8": 0,
|
| 103 |
+
"0.9": 0
|
| 104 |
+
},
|
| 105 |
+
"largest_dup_signed": {
|
| 106 |
+
"0.5": 7,
|
| 107 |
+
"0.6": 7,
|
| 108 |
+
"0.7": 2,
|
| 109 |
+
"0.8": 1,
|
| 110 |
+
"0.9": 1
|
| 111 |
+
},
|
| 112 |
+
"babel_mu1_signed": 0.4506072700023651,
|
| 113 |
+
"n_dup_abs": {
|
| 114 |
+
"0.5": 3913,
|
| 115 |
+
"0.6": 124,
|
| 116 |
+
"0.7": 10,
|
| 117 |
+
"0.8": 0,
|
| 118 |
+
"0.9": 0
|
| 119 |
+
},
|
| 120 |
+
"n_dup_clusters_abs": {
|
| 121 |
+
"0.5": 1787,
|
| 122 |
+
"0.6": 59,
|
| 123 |
+
"0.7": 5,
|
| 124 |
+
"0.8": 0,
|
| 125 |
+
"0.9": 0
|
| 126 |
+
},
|
| 127 |
+
"largest_dup_abs": {
|
| 128 |
+
"0.5": 7,
|
| 129 |
+
"0.6": 7,
|
| 130 |
+
"0.7": 2,
|
| 131 |
+
"0.8": 1,
|
| 132 |
+
"0.9": 1
|
| 133 |
+
},
|
| 134 |
+
"babel_mu1_abs": 0.45388028025627136
|
| 135 |
+
},
|
| 136 |
+
"seed2": {
|
| 137 |
+
"d_sae": 12288,
|
| 138 |
+
"d_in": 768,
|
| 139 |
+
"l0": 126,
|
| 140 |
+
"l0_raw": 126.127159765625,
|
| 141 |
+
"recon_r2": 0.9437536036438203,
|
| 142 |
+
"l1_per_token": 289.6085860579297,
|
| 143 |
+
"stats_source": "stats",
|
| 144 |
+
"n_dead": 0,
|
| 145 |
+
"dead_rate": 0.0,
|
| 146 |
+
"alive_source": "stats",
|
| 147 |
+
"n_dense": 20,
|
| 148 |
+
"dense_rate": 0.0016276041666666667,
|
| 149 |
+
"dense_threshold": 0.1,
|
| 150 |
+
"n_dup_signed": {
|
| 151 |
+
"0.5": 3873,
|
| 152 |
+
"0.6": 21,
|
| 153 |
+
"0.7": 2,
|
| 154 |
+
"0.8": 0,
|
| 155 |
+
"0.9": 0
|
| 156 |
+
},
|
| 157 |
+
"n_dup_clusters_signed": {
|
| 158 |
+
"0.5": 1766,
|
| 159 |
+
"0.6": 8,
|
| 160 |
+
"0.7": 1,
|
| 161 |
+
"0.8": 0,
|
| 162 |
+
"0.9": 0
|
| 163 |
+
},
|
| 164 |
+
"largest_dup_signed": {
|
| 165 |
+
"0.5": 8,
|
| 166 |
+
"0.6": 5,
|
| 167 |
+
"0.7": 2,
|
| 168 |
+
"0.8": 1,
|
| 169 |
+
"0.9": 1
|
| 170 |
+
},
|
| 171 |
+
"babel_mu1_signed": 0.45112404227256775,
|
| 172 |
+
"n_dup_abs": {
|
| 173 |
+
"0.5": 3969,
|
| 174 |
+
"0.6": 125,
|
| 175 |
+
"0.7": 12,
|
| 176 |
+
"0.8": 0,
|
| 177 |
+
"0.9": 0
|
| 178 |
+
},
|
| 179 |
+
"n_dup_clusters_abs": {
|
| 180 |
+
"0.5": 1813,
|
| 181 |
+
"0.6": 60,
|
| 182 |
+
"0.7": 6,
|
| 183 |
+
"0.8": 0,
|
| 184 |
+
"0.9": 0
|
| 185 |
+
},
|
| 186 |
+
"largest_dup_abs": {
|
| 187 |
+
"0.5": 8,
|
| 188 |
+
"0.6": 5,
|
| 189 |
+
"0.7": 2,
|
| 190 |
+
"0.8": 1,
|
| 191 |
+
"0.9": 1
|
| 192 |
+
},
|
| 193 |
+
"babel_mu1_abs": 0.4543568193912506
|
| 194 |
+
}
|
| 195 |
+
},
|
| 196 |
+
"per_pair": {
|
| 197 |
+
"seed0\u2194seed1": {
|
| 198 |
+
"dcs_signed": 0.8446399569511414,
|
| 199 |
+
"dcs_abs": 0.8448325991630554,
|
| 200 |
+
"frac_sign_flipped": 0.00390625,
|
| 201 |
+
"alive_source": "stats",
|
| 202 |
+
"frac_denominator": "d_sae",
|
| 203 |
+
"n_alive_a": 12288,
|
| 204 |
+
"n_alive_b": 12288,
|
| 205 |
+
"matched_cos_signed": {
|
| 206 |
+
"0.5": 0.8421996831893921,
|
| 207 |
+
"0.6": 0.8446352481842041,
|
| 208 |
+
"0.7": 0.8446696400642395,
|
| 209 |
+
"0.8": 0.8446400165557861,
|
| 210 |
+
"0.9": 0.8446400165557861
|
| 211 |
+
},
|
| 212 |
+
"unique_frac_signed": {
|
| 213 |
+
"0.5": 0.8274332682291666,
|
| 214 |
+
"0.6": 0.9991048177083334,
|
| 215 |
+
"0.7": 0.9998779296875,
|
| 216 |
+
"0.8": 1.0,
|
| 217 |
+
"0.9": 1.0
|
| 218 |
+
},
|
| 219 |
+
"dedup_dcs_signed": {
|
| 220 |
+
"0.5": 0.6968640363629675,
|
| 221 |
+
"0.6": 0.8438791456671122,
|
| 222 |
+
"0.7": 0.8445665309773176,
|
| 223 |
+
"0.8": 0.8446400165557861,
|
| 224 |
+
"0.9": 0.8446400165557861
|
| 225 |
+
},
|
| 226 |
+
"matched_frac_signed": {
|
| 227 |
+
"0.5": 0.772705078125,
|
| 228 |
+
"0.6": 0.9130045572916666,
|
| 229 |
+
"0.7": 0.8390299479166666,
|
| 230 |
+
"0.8": 0.719970703125,
|
| 231 |
+
"0.9": 0.5185546875
|
| 232 |
+
},
|
| 233 |
+
"matched_cos_abs": {
|
| 234 |
+
"0.5": 0.8424644470214844,
|
| 235 |
+
"0.6": 0.8453661203384399,
|
| 236 |
+
"0.7": 0.8450070023536682,
|
| 237 |
+
"0.8": 0.8448325991630554,
|
| 238 |
+
"0.9": 0.8448325991630554
|
| 239 |
+
},
|
| 240 |
+
"unique_frac_abs": {
|
| 241 |
+
"0.5": 0.824462890625,
|
| 242 |
+
"0.6": 0.994873046875,
|
| 243 |
+
"0.7": 0.9993896484375,
|
| 244 |
+
"0.8": 1.0,
|
| 245 |
+
"0.9": 1.0
|
| 246 |
+
},
|
| 247 |
+
"dedup_dcs_abs": {
|
| 248 |
+
"0.5": 0.6945806732401252,
|
| 249 |
+
"0.6": 0.8410319678660017,
|
| 250 |
+
"0.7": 0.8444912510094582,
|
| 251 |
+
"0.8": 0.8448325991630554,
|
| 252 |
+
"0.9": 0.8448325991630554
|
| 253 |
+
},
|
| 254 |
+
"matched_frac_abs": {
|
| 255 |
+
"0.5": 0.7703450520833334,
|
| 256 |
+
"0.6": 0.908935546875,
|
| 257 |
+
"0.7": 0.8392740885416666,
|
| 258 |
+
"0.8": 0.7202962239583334,
|
| 259 |
+
"0.9": 0.518798828125
|
| 260 |
+
}
|
| 261 |
+
},
|
| 262 |
+
"seed0\u2194seed2": {
|
| 263 |
+
"dcs_signed": 0.8424331545829773,
|
| 264 |
+
"dcs_abs": 0.8425608277320862,
|
| 265 |
+
"frac_sign_flipped": 0.0032552083333333335,
|
| 266 |
+
"alive_source": "stats",
|
| 267 |
+
"frac_denominator": "d_sae",
|
| 268 |
+
"n_alive_a": 12288,
|
| 269 |
+
"n_alive_b": 12288,
|
| 270 |
+
"matched_cos_signed": {
|
| 271 |
+
"0.5": 0.8382132649421692,
|
| 272 |
+
"0.6": 0.8423312306404114,
|
| 273 |
+
"0.7": 0.8424346446990967,
|
| 274 |
+
"0.8": 0.8424331545829773,
|
| 275 |
+
"0.9": 0.8424331545829773
|
| 276 |
+
},
|
| 277 |
+
"unique_frac_signed": {
|
| 278 |
+
"0.5": 0.8263346354166666,
|
| 279 |
+
"0.6": 0.9989827473958334,
|
| 280 |
+
"0.7": 0.9998779296875,
|
| 281 |
+
"0.8": 1.0,
|
| 282 |
+
"0.9": 1.0
|
| 283 |
+
},
|
| 284 |
+
"dedup_dcs_signed": {
|
| 285 |
+
"0.5": 0.6926446526874012,
|
| 286 |
+
"0.6": 0.8414743670024716,
|
| 287 |
+
"0.7": 0.8423318084387574,
|
| 288 |
+
"0.8": 0.8424331545829773,
|
| 289 |
+
"0.9": 0.8424331545829773
|
| 290 |
+
},
|
| 291 |
+
"matched_frac_signed": {
|
| 292 |
+
"0.5": 0.77197265625,
|
| 293 |
+
"0.6": 0.9117024739583334,
|
| 294 |
+
"0.7": 0.83349609375,
|
| 295 |
+
"0.8": 0.7119140625,
|
| 296 |
+
"0.9": 0.5091959635416666
|
| 297 |
+
},
|
| 298 |
+
"matched_cos_abs": {
|
| 299 |
+
"0.5": 0.8384155631065369,
|
| 300 |
+
"0.6": 0.8430929183959961,
|
| 301 |
+
"0.7": 0.8426738381385803,
|
| 302 |
+
"0.8": 0.8425607681274414,
|
| 303 |
+
"0.9": 0.8425607681274414
|
| 304 |
+
},
|
| 305 |
+
"unique_frac_abs": {
|
| 306 |
+
"0.5": 0.8232421875,
|
| 307 |
+
"0.6": 0.994873046875,
|
| 308 |
+
"0.7": 0.9993489583333334,
|
| 309 |
+
"0.8": 1.0,
|
| 310 |
+
"0.9": 1.0
|
| 311 |
+
},
|
| 312 |
+
"dedup_dcs_abs": {
|
| 313 |
+
"0.5": 0.6902190622058697,
|
| 314 |
+
"0.6": 0.8387704205233604,
|
| 315 |
+
"0.7": 0.8421252223585423,
|
| 316 |
+
"0.8": 0.8425607681274414,
|
| 317 |
+
"0.9": 0.8425607681274414
|
| 318 |
+
},
|
| 319 |
+
"matched_frac_abs": {
|
| 320 |
+
"0.5": 0.7692057291666666,
|
| 321 |
+
"0.6": 0.9073079427083334,
|
| 322 |
+
"0.7": 0.8330891927083334,
|
| 323 |
+
"0.8": 0.711669921875,
|
| 324 |
+
"0.9": 0.5091145833333334
|
| 325 |
+
}
|
| 326 |
+
},
|
| 327 |
+
"seed1\u2194seed2": {
|
| 328 |
+
"dcs_signed": 0.843366801738739,
|
| 329 |
+
"dcs_abs": 0.8435181975364685,
|
| 330 |
+
"frac_sign_flipped": 0.0035807291666666665,
|
| 331 |
+
"alive_source": "stats",
|
| 332 |
+
"frac_denominator": "d_sae",
|
| 333 |
+
"n_alive_a": 12288,
|
| 334 |
+
"n_alive_b": 12288,
|
| 335 |
+
"matched_cos_signed": {
|
| 336 |
+
"0.5": 0.8385906219482422,
|
| 337 |
+
"0.6": 0.8433858156204224,
|
| 338 |
+
"0.7": 0.8433728814125061,
|
| 339 |
+
"0.8": 0.843366801738739,
|
| 340 |
+
"0.9": 0.843366801738739
|
| 341 |
+
},
|
| 342 |
+
"unique_frac_signed": {
|
| 343 |
+
"0.5": 0.8296305338541666,
|
| 344 |
+
"0.6": 0.9990641276041666,
|
| 345 |
+
"0.7": 0.9999186197916666,
|
| 346 |
+
"0.8": 1.0,
|
| 347 |
+
"0.9": 1.0
|
| 348 |
+
},
|
| 349 |
+
"dedup_dcs_signed": {
|
| 350 |
+
"0.5": 0.6957203853720177,
|
| 351 |
+
"0.6": 0.8425965141165458,
|
| 352 |
+
"0.7": 0.8433042475517141,
|
| 353 |
+
"0.8": 0.843366801738739,
|
| 354 |
+
"0.9": 0.843366801738739
|
| 355 |
+
},
|
| 356 |
+
"matched_frac_signed": {
|
| 357 |
+
"0.5": 0.773681640625,
|
| 358 |
+
"0.6": 0.9065755208333334,
|
| 359 |
+
"0.7": 0.8380533854166666,
|
| 360 |
+
"0.8": 0.7194010416666666,
|
| 361 |
+
"0.9": 0.5166829427083334
|
| 362 |
+
},
|
| 363 |
+
"matched_cos_abs": {
|
| 364 |
+
"0.5": 0.8398551940917969,
|
| 365 |
+
"0.6": 0.8440368175506592,
|
| 366 |
+
"0.7": 0.8436010479927063,
|
| 367 |
+
"0.8": 0.8435182571411133,
|
| 368 |
+
"0.9": 0.8435182571411133
|
| 369 |
+
},
|
| 370 |
+
"unique_frac_abs": {
|
| 371 |
+
"0.5": 0.8257649739583334,
|
| 372 |
+
"0.6": 0.9947102864583334,
|
| 373 |
+
"0.7": 0.9995524088541666,
|
| 374 |
+
"0.8": 1.0,
|
| 375 |
+
"0.9": 1.0
|
| 376 |
+
},
|
| 377 |
+
"dedup_dcs_abs": {
|
| 378 |
+
"0.5": 0.6935230024779837,
|
| 379 |
+
"0.6": 0.8395721045671962,
|
| 380 |
+
"0.7": 0.843223459633009,
|
| 381 |
+
"0.8": 0.8435182571411133,
|
| 382 |
+
"0.9": 0.8435182571411133
|
| 383 |
+
},
|
| 384 |
+
"matched_frac_abs": {
|
| 385 |
+
"0.5": 0.7705891927083334,
|
| 386 |
+
"0.6": 0.904541015625,
|
| 387 |
+
"0.7": 0.8382161458333334,
|
| 388 |
+
"0.8": 0.7191569010416666,
|
| 389 |
+
"0.9": 0.5166829427083334
|
| 390 |
+
}
|
| 391 |
+
}
|
| 392 |
+
},
|
| 393 |
+
"summary": {
|
| 394 |
+
"n_seeds": 3,
|
| 395 |
+
"n_pairs": 3,
|
| 396 |
+
"report_band": [
|
| 397 |
+
0.6,
|
| 398 |
+
0.7,
|
| 399 |
+
0.8,
|
| 400 |
+
0.9
|
| 401 |
+
],
|
| 402 |
+
"n_dead_mean": 0,
|
| 403 |
+
"n_dead_std": 0.0,
|
| 404 |
+
"n_dense_mean": 19,
|
| 405 |
+
"n_dense_std": 0.5773502691896258,
|
| 406 |
+
"l0_mean": 126,
|
| 407 |
+
"l0_std": 0.287502117670696,
|
| 408 |
+
"recon_r2_mean": 0.9437377744284445,
|
| 409 |
+
"recon_r2_std": 3.900158063678982e-05,
|
| 410 |
+
"l1_per_token_mean": 289.5022045182873,
|
| 411 |
+
"l1_per_token_std": 0.1129700313896754,
|
| 412 |
+
"babel_mu1_signed_mean": 0.4512283504009247,
|
| 413 |
+
"babel_mu1_signed_std": 0.0006792678271853955,
|
| 414 |
+
"n_dup_signed_per_threshold": {
|
| 415 |
+
"0.5": 3885.6666666666665,
|
| 416 |
+
"0.6": 18.0,
|
| 417 |
+
"0.7": 2.3333333333333335,
|
| 418 |
+
"0.8": 0.0,
|
| 419 |
+
"0.9": 0.0
|
| 420 |
+
},
|
| 421 |
+
"n_dup_signed_band": 5.083333333333333,
|
| 422 |
+
"n_dup_signed_band_std": 0.9464847243000456,
|
| 423 |
+
"n_dup_clusters_signed_per_threshold": {
|
| 424 |
+
"0.5": 1769.6666666666667,
|
| 425 |
+
"0.6": 6.333333333333333,
|
| 426 |
+
"0.7": 1.0,
|
| 427 |
+
"0.8": 0.0,
|
| 428 |
+
"0.9": 0.0
|
| 429 |
+
},
|
| 430 |
+
"n_dup_clusters_signed_band": 1.8333333333333333,
|
| 431 |
+
"n_dup_clusters_signed_band_std": 0.5204164998665332,
|
| 432 |
+
"largest_dup_signed_per_threshold": {
|
| 433 |
+
"0.5": 7.666666666666667,
|
| 434 |
+
"0.6": 5.333333333333333,
|
| 435 |
+
"0.7": 2.3333333333333335,
|
| 436 |
+
"0.8": 1.0,
|
| 437 |
+
"0.9": 1.0
|
| 438 |
+
},
|
| 439 |
+
"largest_dup_signed_band": 2.4166666666666665,
|
| 440 |
+
"largest_dup_signed_band_std": 0.2886751345948129,
|
| 441 |
+
"dcs_signed_mean": 0.8434799710909525,
|
| 442 |
+
"dcs_signed_std": 0.0011077453000324367,
|
| 443 |
+
"dedup_dcs_signed_per_threshold": {
|
| 444 |
+
"0.5": 0.6950763581407955,
|
| 445 |
+
"0.6": 0.8426500089287098,
|
| 446 |
+
"0.7": 0.8434008623225964,
|
| 447 |
+
"0.8": 0.8434799909591675,
|
| 448 |
+
"0.9": 0.8434799909591675
|
| 449 |
+
},
|
| 450 |
+
"dedup_dcs_signed_band": 0.8432527132924102,
|
| 451 |
+
"dedup_dcs_signed_band_std": 0.001134584968802106,
|
| 452 |
+
"matched_cos_signed_per_threshold": {
|
| 453 |
+
"0.5": 0.8396678566932678,
|
| 454 |
+
"0.6": 0.8434507648150126,
|
| 455 |
+
"0.7": 0.8434923887252808,
|
| 456 |
+
"0.8": 0.8434799909591675,
|
| 457 |
+
"0.9": 0.8434799909591675
|
| 458 |
+
},
|
| 459 |
+
"matched_cos_signed_band": 0.843475783864657,
|
| 460 |
+
"matched_cos_signed_band_std": 0.0011226214664995392,
|
| 461 |
+
"unique_frac_signed_per_threshold": {
|
| 462 |
+
"0.5": 0.8277994791666666,
|
| 463 |
+
"0.6": 0.9990505642361112,
|
| 464 |
+
"0.7": 0.9998914930555555,
|
| 465 |
+
"0.8": 1.0,
|
| 466 |
+
"0.9": 1.0
|
| 467 |
+
},
|
| 468 |
+
"unique_frac_signed_band": 0.9997355143229166,
|
| 469 |
+
"unique_frac_signed_band_std": 1.7619331945452135e-05,
|
| 470 |
+
"matched_frac_signed_per_threshold": {
|
| 471 |
+
"0.5": 0.7727864583333334,
|
| 472 |
+
"0.6": 0.9104275173611112,
|
| 473 |
+
"0.7": 0.8368598090277777,
|
| 474 |
+
"0.8": 0.7170952690972222,
|
| 475 |
+
"0.9": 0.5148111979166666
|
| 476 |
+
},
|
| 477 |
+
"matched_frac_signed_band": 0.7447984483506944,
|
| 478 |
+
"matched_frac_signed_band_std": 0.003049202308208975,
|
| 479 |
+
"babel_mu1_abs_mean": 0.45445024967193604,
|
| 480 |
+
"babel_mu1_abs_std": 0.0006219700592711387,
|
| 481 |
+
"n_dup_abs_per_threshold": {
|
| 482 |
+
"0.5": 3967.0,
|
| 483 |
+
"0.6": 122.0,
|
| 484 |
+
"0.7": 13.666666666666666,
|
| 485 |
+
"0.8": 0.0,
|
| 486 |
+
"0.9": 0.0
|
| 487 |
+
},
|
| 488 |
+
"n_dup_abs_band": 33.916666666666664,
|
| 489 |
+
"n_dup_abs_band_std": 0.38188130791298663,
|
| 490 |
+
"n_dup_clusters_abs_per_threshold": {
|
| 491 |
+
"0.5": 1810.3333333333333,
|
| 492 |
+
"0.6": 58.333333333333336,
|
| 493 |
+
"0.7": 6.666666666666667,
|
| 494 |
+
"0.8": 0.0,
|
| 495 |
+
"0.9": 0.0
|
| 496 |
+
},
|
| 497 |
+
"n_dup_clusters_abs_band": 16.25,
|
| 498 |
+
"n_dup_clusters_abs_band_std": 0.25,
|
| 499 |
+
"largest_dup_abs_per_threshold": {
|
| 500 |
+
"0.5": 7.666666666666667,
|
| 501 |
+
"0.6": 5.333333333333333,
|
| 502 |
+
"0.7": 2.3333333333333335,
|
| 503 |
+
"0.8": 1.0,
|
| 504 |
+
"0.9": 1.0
|
| 505 |
+
},
|
| 506 |
+
"largest_dup_abs_band": 2.4166666666666665,
|
| 507 |
+
"largest_dup_abs_band_std": 0.2886751345948129,
|
| 508 |
+
"dcs_abs_mean": 0.84363720814387,
|
| 509 |
+
"dcs_abs_std": 0.0011405520602530892,
|
| 510 |
+
"dedup_dcs_abs_per_threshold": {
|
| 511 |
+
"0.5": 0.6927742459746596,
|
| 512 |
+
"0.6": 0.8397914976521861,
|
| 513 |
+
"0.7": 0.8432799776670032,
|
| 514 |
+
"0.8": 0.84363720814387,
|
| 515 |
+
"0.9": 0.84363720814387
|
| 516 |
+
},
|
| 517 |
+
"dedup_dcs_abs_band": 0.8425864729017323,
|
| 518 |
+
"dedup_dcs_abs_band_std": 0.0011517895005007416,
|
| 519 |
+
"matched_cos_abs_per_threshold": {
|
| 520 |
+
"0.5": 0.8402450680732727,
|
| 521 |
+
"0.6": 0.8441652854283651,
|
| 522 |
+
"0.7": 0.843760629494985,
|
| 523 |
+
"0.8": 0.84363720814387,
|
| 524 |
+
"0.9": 0.84363720814387
|
| 525 |
+
},
|
| 526 |
+
"matched_cos_abs_band": 0.8438000828027725,
|
| 527 |
+
"matched_cos_abs_band_std": 0.001149408075698793,
|
| 528 |
+
"unique_frac_abs_per_threshold": {
|
| 529 |
+
"0.5": 0.8244900173611112,
|
| 530 |
+
"0.6": 0.9948187934027778,
|
| 531 |
+
"0.7": 0.9994303385416666,
|
| 532 |
+
"0.8": 1.0,
|
| 533 |
+
"0.9": 1.0
|
| 534 |
+
},
|
| 535 |
+
"unique_frac_abs_band": 0.9985622829861112,
|
| 536 |
+
"unique_frac_abs_band_std": 5.873110648473362e-06,
|
| 537 |
+
"matched_frac_abs_per_threshold": {
|
| 538 |
+
"0.5": 0.7700466579861112,
|
| 539 |
+
"0.6": 0.9069281684027778,
|
| 540 |
+
"0.7": 0.8368598090277778,
|
| 541 |
+
"0.8": 0.717041015625,
|
| 542 |
+
"0.9": 0.514865451388889
|
| 543 |
+
},
|
| 544 |
+
"matched_frac_abs_band": 0.743923611111111,
|
| 545 |
+
"matched_frac_abs_band_std": 0.0033253011024323635,
|
| 546 |
+
"alive_source": "stats",
|
| 547 |
+
"frac_denominator": "d_sae",
|
| 548 |
+
"d_sae": 12288
|
| 549 |
+
}
|
| 550 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 16,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 0,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 4.5,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 10,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_samples": 20000,
|
| 3 |
+
"n_positions": 20480000,
|
| 4 |
+
"l0_per_token_0": 126.4695044921875,
|
| 5 |
+
"l0_per_token_0.001": 126.46924731445313,
|
| 6 |
+
"l0_per_token_0.01": 126.46831743164063,
|
| 7 |
+
"l0_per_token_0.05": 126.46364501953126,
|
| 8 |
+
"l0_per_token_0.1": 126.454286328125,
|
| 9 |
+
"l0_per_token_0.5": 111.81361259765625,
|
| 10 |
+
"l0_per_token_1.0": 79.31354155273438,
|
| 11 |
+
"l1_per_token": 289.3836344397387,
|
| 12 |
+
"r2_score": 0.9436933468059865,
|
| 13 |
+
"l2_loss": 1.3205280686668774
|
| 14 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "fineweb",
|
| 3 |
+
"split": "train",
|
| 4 |
+
"hf_dataset": "HuggingFaceFW/fineweb",
|
| 5 |
+
"hf_dataset_config": null,
|
| 6 |
+
"hf_dataset_text_field": "text",
|
| 7 |
+
"context_length": 1024,
|
| 8 |
+
"num_samples_requested": 20000,
|
| 9 |
+
"num_samples_actual": 20000,
|
| 10 |
+
"batch_seqs": 32,
|
| 11 |
+
"stats_top_k": 100,
|
| 12 |
+
"stream_seed": 0,
|
| 13 |
+
"sparsity_thresholds": [
|
| 14 |
+
0.001,
|
| 15 |
+
0.01,
|
| 16 |
+
0.05,
|
| 17 |
+
0.1,
|
| 18 |
+
0.5,
|
| 19 |
+
1.0
|
| 20 |
+
],
|
| 21 |
+
"model_hf_name": "openai-community/gpt2",
|
| 22 |
+
"hook_target_name": "transformer.h.8",
|
| 23 |
+
"tokenizer_hf_name": "openai-community/gpt2",
|
| 24 |
+
"d_sae": 12288,
|
| 25 |
+
"d_in": 768
|
| 26 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 16,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 1,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 4.5,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 10,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_samples": 20000,
|
| 3 |
+
"n_positions": 20480000,
|
| 4 |
+
"l0_per_token_0": 125.89824155273438,
|
| 5 |
+
"l0_per_token_0.001": 125.89810434570313,
|
| 6 |
+
"l0_per_token_0.01": 125.89738203125,
|
| 7 |
+
"l0_per_token_0.05": 125.89346733398438,
|
| 8 |
+
"l0_per_token_0.1": 125.88476430664062,
|
| 9 |
+
"l0_per_token_0.5": 111.54088544921875,
|
| 10 |
+
"l0_per_token_1.0": 79.13532709960937,
|
| 11 |
+
"l1_per_token": 289.51439305719344,
|
| 12 |
+
"r2_score": 0.9437663728355269,
|
| 13 |
+
"l2_loss": 1.3188154305276674
|
| 14 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "fineweb",
|
| 3 |
+
"split": "train",
|
| 4 |
+
"hf_dataset": "HuggingFaceFW/fineweb",
|
| 5 |
+
"hf_dataset_config": null,
|
| 6 |
+
"hf_dataset_text_field": "text",
|
| 7 |
+
"context_length": 1024,
|
| 8 |
+
"num_samples_requested": 20000,
|
| 9 |
+
"num_samples_actual": 20000,
|
| 10 |
+
"batch_seqs": 32,
|
| 11 |
+
"stats_top_k": 100,
|
| 12 |
+
"stream_seed": 0,
|
| 13 |
+
"sparsity_thresholds": [
|
| 14 |
+
0.001,
|
| 15 |
+
0.01,
|
| 16 |
+
0.05,
|
| 17 |
+
0.1,
|
| 18 |
+
0.5,
|
| 19 |
+
1.0
|
| 20 |
+
],
|
| 21 |
+
"model_hf_name": "openai-community/gpt2",
|
| 22 |
+
"hook_target_name": "transformer.h.8",
|
| 23 |
+
"tokenizer_hf_name": "openai-community/gpt2",
|
| 24 |
+
"d_sae": 12288,
|
| 25 |
+
"d_in": 768
|
| 26 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 16,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 2,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 4.5,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 10,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_samples": 20000,
|
| 3 |
+
"n_positions": 20480000,
|
| 4 |
+
"l0_per_token_0": 126.127159765625,
|
| 5 |
+
"l0_per_token_0.001": 126.12707016601563,
|
| 6 |
+
"l0_per_token_0.01": 126.12630732421874,
|
| 7 |
+
"l0_per_token_0.05": 126.12196870117188,
|
| 8 |
+
"l0_per_token_0.1": 126.11321782226563,
|
| 9 |
+
"l0_per_token_0.5": 111.70350498046875,
|
| 10 |
+
"l0_per_token_1.0": 79.29182919921875,
|
| 11 |
+
"l1_per_token": 289.6085860579297,
|
| 12 |
+
"r2_score": 0.9437536036438203,
|
| 13 |
+
"l2_loss": 1.3191148991535948
|
| 14 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "fineweb",
|
| 3 |
+
"split": "train",
|
| 4 |
+
"hf_dataset": "HuggingFaceFW/fineweb",
|
| 5 |
+
"hf_dataset_config": null,
|
| 6 |
+
"hf_dataset_text_field": "text",
|
| 7 |
+
"context_length": 1024,
|
| 8 |
+
"num_samples_requested": 20000,
|
| 9 |
+
"num_samples_actual": 20000,
|
| 10 |
+
"batch_seqs": 32,
|
| 11 |
+
"stats_top_k": 100,
|
| 12 |
+
"stream_seed": 0,
|
| 13 |
+
"sparsity_thresholds": [
|
| 14 |
+
0.001,
|
| 15 |
+
0.01,
|
| 16 |
+
0.05,
|
| 17 |
+
0.1,
|
| 18 |
+
0.5,
|
| 19 |
+
1.0
|
| 20 |
+
],
|
| 21 |
+
"model_hf_name": "openai-community/gpt2",
|
| 22 |
+
"hook_target_name": "transformer.h.8",
|
| 23 |
+
"tokenizer_hf_name": "openai-community/gpt2",
|
| 24 |
+
"d_sae": 12288,
|
| 25 |
+
"d_in": 768
|
| 26 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_32/code_metrics_step_final.json
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": 1,
|
| 3 |
+
"step": "final",
|
| 4 |
+
"n_tokens": 600000,
|
| 5 |
+
"thresholds": [
|
| 6 |
+
0.5,
|
| 7 |
+
0.6,
|
| 8 |
+
0.7,
|
| 9 |
+
0.8,
|
| 10 |
+
0.9
|
| 11 |
+
],
|
| 12 |
+
"seeds": [
|
| 13 |
+
"seed0",
|
| 14 |
+
"seed1",
|
| 15 |
+
"seed2"
|
| 16 |
+
],
|
| 17 |
+
"per_seed": {
|
| 18 |
+
"seed0": {
|
| 19 |
+
"l0": 34.11666833333334,
|
| 20 |
+
"recon_r2": 0.8986679004156218,
|
| 21 |
+
"l1_per_token": 186.22070988001735
|
| 22 |
+
},
|
| 23 |
+
"seed1": {
|
| 24 |
+
"l0": 34.10902166666666,
|
| 25 |
+
"recon_r2": 0.8987894297283623,
|
| 26 |
+
"l1_per_token": 185.72599150414578
|
| 27 |
+
},
|
| 28 |
+
"seed2": {
|
| 29 |
+
"l0": 34.240206666666666,
|
| 30 |
+
"recon_r2": 0.8987818586148901,
|
| 31 |
+
"l1_per_token": 186.73523437882213
|
| 32 |
+
}
|
| 33 |
+
},
|
| 34 |
+
"per_pair": {
|
| 35 |
+
"seed0\u2194seed1": {
|
| 36 |
+
"alive_source": "stats",
|
| 37 |
+
"mcc_signed": 0.7809844014446874,
|
| 38 |
+
"mcc_abs": 0.780985660585697,
|
| 39 |
+
"act_iou": 0.5074479851575096,
|
| 40 |
+
"act_iou_n_pairs": 12265,
|
| 41 |
+
"input_r2": 0.9863562707505128,
|
| 42 |
+
"matched_mcc_signed": {
|
| 43 |
+
"0.6": 0.8304022051494929,
|
| 44 |
+
"0.7": 0.857498462100996,
|
| 45 |
+
"0.8": 0.8958479882608926,
|
| 46 |
+
"0.9": 0.9414620631216186
|
| 47 |
+
},
|
| 48 |
+
"matched_mcc_abs": {
|
| 49 |
+
"0.6": 0.8308284407037294,
|
| 50 |
+
"0.7": 0.8573308349313363,
|
| 51 |
+
"0.8": 0.895670010459787,
|
| 52 |
+
"0.9": 0.9414558876475153
|
| 53 |
+
}
|
| 54 |
+
},
|
| 55 |
+
"seed0\u2194seed2": {
|
| 56 |
+
"alive_source": "stats",
|
| 57 |
+
"mcc_signed": 0.7779173925424955,
|
| 58 |
+
"mcc_abs": 0.7779183138998308,
|
| 59 |
+
"act_iou": 0.5054622423786899,
|
| 60 |
+
"act_iou_n_pairs": 12271,
|
| 61 |
+
"input_r2": 0.9863696984638582,
|
| 62 |
+
"matched_mcc_signed": {
|
| 63 |
+
"0.6": 0.8251178698395323,
|
| 64 |
+
"0.7": 0.8545442564293723,
|
| 65 |
+
"0.8": 0.8941268562305976,
|
| 66 |
+
"0.9": 0.9405185795513432
|
| 67 |
+
},
|
| 68 |
+
"matched_mcc_abs": {
|
| 69 |
+
"0.6": 0.8252502187072939,
|
| 70 |
+
"0.7": 0.8545884774945348,
|
| 71 |
+
"0.8": 0.8943297597748052,
|
| 72 |
+
"0.9": 0.9404768194422161
|
| 73 |
+
}
|
| 74 |
+
},
|
| 75 |
+
"seed1\u2194seed2": {
|
| 76 |
+
"alive_source": "stats",
|
| 77 |
+
"mcc_signed": 0.7805839115016303,
|
| 78 |
+
"mcc_abs": 0.7805845164009844,
|
| 79 |
+
"act_iou": 0.5090701067602436,
|
| 80 |
+
"act_iou_n_pairs": 12268,
|
| 81 |
+
"input_r2": 0.9864737100906098,
|
| 82 |
+
"matched_mcc_signed": {
|
| 83 |
+
"0.6": 0.8274534585829221,
|
| 84 |
+
"0.7": 0.8567122593646093,
|
| 85 |
+
"0.8": 0.8951611853298442,
|
| 86 |
+
"0.9": 0.9415611179795759
|
| 87 |
+
},
|
| 88 |
+
"matched_mcc_abs": {
|
| 89 |
+
"0.6": 0.8272814411540946,
|
| 90 |
+
"0.7": 0.8561547899705673,
|
| 91 |
+
"0.8": 0.8952639714169829,
|
| 92 |
+
"0.9": 0.9415610600249489
|
| 93 |
+
}
|
| 94 |
+
}
|
| 95 |
+
},
|
| 96 |
+
"summary": {
|
| 97 |
+
"n_seeds": 3,
|
| 98 |
+
"n_pairs": 3,
|
| 99 |
+
"report_band": [
|
| 100 |
+
0.6,
|
| 101 |
+
0.7,
|
| 102 |
+
0.8,
|
| 103 |
+
0.9
|
| 104 |
+
],
|
| 105 |
+
"l0_mean": 34.155298888888886,
|
| 106 |
+
"l0_std": 0.0736316231190653,
|
| 107 |
+
"recon_r2_mean": 0.8987463962529582,
|
| 108 |
+
"recon_r2_std": 6.808471046528173e-05,
|
| 109 |
+
"l1_per_token_mean": 186.2273119209951,
|
| 110 |
+
"l1_per_token_std": 0.5046538271231552,
|
| 111 |
+
"input_r2_mean": 0.9863998931016603,
|
| 112 |
+
"input_r2_std": 6.427897606995591e-05,
|
| 113 |
+
"mcc_signed_mean": 0.779828568496271,
|
| 114 |
+
"mcc_signed_std": 0.0016671962072113684,
|
| 115 |
+
"mcc_abs_mean": 0.7798294969621707,
|
| 116 |
+
"mcc_abs_std": 0.0016672416404239093,
|
| 117 |
+
"matched_mcc_signed_per_threshold": {
|
| 118 |
+
"0.6": 0.8276578445239825,
|
| 119 |
+
"0.7": 0.8562516592983259,
|
| 120 |
+
"0.8": 0.8950453432737783,
|
| 121 |
+
"0.9": 0.9411805868841793
|
| 122 |
+
},
|
| 123 |
+
"matched_mcc_signed_band": 0.8800338584950664,
|
| 124 |
+
"matched_mcc_abs_per_threshold": {
|
| 125 |
+
"0.6": 0.8277867001883726,
|
| 126 |
+
"0.7": 0.8560247007988128,
|
| 127 |
+
"0.8": 0.8950879138838584,
|
| 128 |
+
"0.9": 0.9411645890382268
|
| 129 |
+
},
|
| 130 |
+
"matched_mcc_abs_band": 0.8800159759773176,
|
| 131 |
+
"act_iou_mean": 0.5073267780988143,
|
| 132 |
+
"act_iou_std": 0.001806983594897617,
|
| 133 |
+
"act_iou_n_pairs": 12268.0,
|
| 134 |
+
"alive_source": "stats"
|
| 135 |
+
}
|
| 136 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_32/dict_metrics_step_final.json
ADDED
|
@@ -0,0 +1,550 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": 2,
|
| 3 |
+
"step": "final",
|
| 4 |
+
"linkage": "complete",
|
| 5 |
+
"thresholds": [
|
| 6 |
+
0.5,
|
| 7 |
+
0.6,
|
| 8 |
+
0.7,
|
| 9 |
+
0.8,
|
| 10 |
+
0.9
|
| 11 |
+
],
|
| 12 |
+
"seeds": [
|
| 13 |
+
"seed0",
|
| 14 |
+
"seed1",
|
| 15 |
+
"seed2"
|
| 16 |
+
],
|
| 17 |
+
"per_seed": {
|
| 18 |
+
"seed0": {
|
| 19 |
+
"d_sae": 12288,
|
| 20 |
+
"d_in": 768,
|
| 21 |
+
"l0": 34,
|
| 22 |
+
"l0_raw": 34.09826708984375,
|
| 23 |
+
"recon_r2": 0.8984899544332474,
|
| 24 |
+
"l1_per_token": 186.09178804915646,
|
| 25 |
+
"stats_source": "stats",
|
| 26 |
+
"n_dead": 32,
|
| 27 |
+
"dead_rate": 0.0026041666666666665,
|
| 28 |
+
"alive_source": "stats",
|
| 29 |
+
"n_dense": 0,
|
| 30 |
+
"dense_rate": 0.0,
|
| 31 |
+
"dense_threshold": 0.1,
|
| 32 |
+
"n_dup_signed": {
|
| 33 |
+
"0.5": 5875,
|
| 34 |
+
"0.6": 123,
|
| 35 |
+
"0.7": 0,
|
| 36 |
+
"0.8": 0,
|
| 37 |
+
"0.9": 0
|
| 38 |
+
},
|
| 39 |
+
"n_dup_clusters_signed": {
|
| 40 |
+
"0.5": 2667,
|
| 41 |
+
"0.6": 60,
|
| 42 |
+
"0.7": 0,
|
| 43 |
+
"0.8": 0,
|
| 44 |
+
"0.9": 0
|
| 45 |
+
},
|
| 46 |
+
"largest_dup_signed": {
|
| 47 |
+
"0.5": 6,
|
| 48 |
+
"0.6": 3,
|
| 49 |
+
"0.7": 1,
|
| 50 |
+
"0.8": 1,
|
| 51 |
+
"0.9": 1
|
| 52 |
+
},
|
| 53 |
+
"babel_mu1_signed": 0.48740267753601074,
|
| 54 |
+
"n_dup_abs": {
|
| 55 |
+
"0.5": 5894,
|
| 56 |
+
"0.6": 193,
|
| 57 |
+
"0.7": 12,
|
| 58 |
+
"0.8": 0,
|
| 59 |
+
"0.9": 0
|
| 60 |
+
},
|
| 61 |
+
"n_dup_clusters_abs": {
|
| 62 |
+
"0.5": 2671,
|
| 63 |
+
"0.6": 95,
|
| 64 |
+
"0.7": 6,
|
| 65 |
+
"0.8": 0,
|
| 66 |
+
"0.9": 0
|
| 67 |
+
},
|
| 68 |
+
"largest_dup_abs": {
|
| 69 |
+
"0.5": 6,
|
| 70 |
+
"0.6": 3,
|
| 71 |
+
"0.7": 2,
|
| 72 |
+
"0.8": 1,
|
| 73 |
+
"0.9": 1
|
| 74 |
+
},
|
| 75 |
+
"babel_mu1_abs": 0.4891930818557739
|
| 76 |
+
},
|
| 77 |
+
"seed1": {
|
| 78 |
+
"d_sae": 12288,
|
| 79 |
+
"d_in": 768,
|
| 80 |
+
"l0": 34,
|
| 81 |
+
"l0_raw": 34.087190966796875,
|
| 82 |
+
"recon_r2": 0.8986077841689825,
|
| 83 |
+
"l1_per_token": 185.59309232967692,
|
| 84 |
+
"stats_source": "stats",
|
| 85 |
+
"n_dead": 38,
|
| 86 |
+
"dead_rate": 0.0030924479166666665,
|
| 87 |
+
"alive_source": "stats",
|
| 88 |
+
"n_dense": 0,
|
| 89 |
+
"dense_rate": 0.0,
|
| 90 |
+
"dense_threshold": 0.1,
|
| 91 |
+
"n_dup_signed": {
|
| 92 |
+
"0.5": 5935,
|
| 93 |
+
"0.6": 127,
|
| 94 |
+
"0.7": 0,
|
| 95 |
+
"0.8": 0,
|
| 96 |
+
"0.9": 0
|
| 97 |
+
},
|
| 98 |
+
"n_dup_clusters_signed": {
|
| 99 |
+
"0.5": 2682,
|
| 100 |
+
"0.6": 63,
|
| 101 |
+
"0.7": 0,
|
| 102 |
+
"0.8": 0,
|
| 103 |
+
"0.9": 0
|
| 104 |
+
},
|
| 105 |
+
"largest_dup_signed": {
|
| 106 |
+
"0.5": 6,
|
| 107 |
+
"0.6": 3,
|
| 108 |
+
"0.7": 1,
|
| 109 |
+
"0.8": 1,
|
| 110 |
+
"0.9": 1
|
| 111 |
+
},
|
| 112 |
+
"babel_mu1_signed": 0.4871804714202881,
|
| 113 |
+
"n_dup_abs": {
|
| 114 |
+
"0.5": 5942,
|
| 115 |
+
"0.6": 195,
|
| 116 |
+
"0.7": 16,
|
| 117 |
+
"0.8": 2,
|
| 118 |
+
"0.9": 0
|
| 119 |
+
},
|
| 120 |
+
"n_dup_clusters_abs": {
|
| 121 |
+
"0.5": 2684,
|
| 122 |
+
"0.6": 97,
|
| 123 |
+
"0.7": 8,
|
| 124 |
+
"0.8": 1,
|
| 125 |
+
"0.9": 0
|
| 126 |
+
},
|
| 127 |
+
"largest_dup_abs": {
|
| 128 |
+
"0.5": 6,
|
| 129 |
+
"0.6": 3,
|
| 130 |
+
"0.7": 2,
|
| 131 |
+
"0.8": 2,
|
| 132 |
+
"0.9": 1
|
| 133 |
+
},
|
| 134 |
+
"babel_mu1_abs": 0.4889828860759735
|
| 135 |
+
},
|
| 136 |
+
"seed2": {
|
| 137 |
+
"d_sae": 12288,
|
| 138 |
+
"d_in": 768,
|
| 139 |
+
"l0": 34,
|
| 140 |
+
"l0_raw": 34.21214306640625,
|
| 141 |
+
"recon_r2": 0.898614796428707,
|
| 142 |
+
"l1_per_token": 186.597777168898,
|
| 143 |
+
"stats_source": "stats",
|
| 144 |
+
"n_dead": 25,
|
| 145 |
+
"dead_rate": 0.0020345052083333335,
|
| 146 |
+
"alive_source": "stats",
|
| 147 |
+
"n_dense": 0,
|
| 148 |
+
"dense_rate": 0.0,
|
| 149 |
+
"dense_threshold": 0.1,
|
| 150 |
+
"n_dup_signed": {
|
| 151 |
+
"0.5": 5966,
|
| 152 |
+
"0.6": 142,
|
| 153 |
+
"0.7": 0,
|
| 154 |
+
"0.8": 0,
|
| 155 |
+
"0.9": 0
|
| 156 |
+
},
|
| 157 |
+
"n_dup_clusters_signed": {
|
| 158 |
+
"0.5": 2699,
|
| 159 |
+
"0.6": 70,
|
| 160 |
+
"0.7": 0,
|
| 161 |
+
"0.8": 0,
|
| 162 |
+
"0.9": 0
|
| 163 |
+
},
|
| 164 |
+
"largest_dup_signed": {
|
| 165 |
+
"0.5": 7,
|
| 166 |
+
"0.6": 3,
|
| 167 |
+
"0.7": 1,
|
| 168 |
+
"0.8": 1,
|
| 169 |
+
"0.9": 1
|
| 170 |
+
},
|
| 171 |
+
"babel_mu1_signed": 0.48869165778160095,
|
| 172 |
+
"n_dup_abs": {
|
| 173 |
+
"0.5": 5969,
|
| 174 |
+
"0.6": 212,
|
| 175 |
+
"0.7": 14,
|
| 176 |
+
"0.8": 0,
|
| 177 |
+
"0.9": 0
|
| 178 |
+
},
|
| 179 |
+
"n_dup_clusters_abs": {
|
| 180 |
+
"0.5": 2699,
|
| 181 |
+
"0.6": 105,
|
| 182 |
+
"0.7": 7,
|
| 183 |
+
"0.8": 0,
|
| 184 |
+
"0.9": 0
|
| 185 |
+
},
|
| 186 |
+
"largest_dup_abs": {
|
| 187 |
+
"0.5": 7,
|
| 188 |
+
"0.6": 3,
|
| 189 |
+
"0.7": 2,
|
| 190 |
+
"0.8": 1,
|
| 191 |
+
"0.9": 1
|
| 192 |
+
},
|
| 193 |
+
"babel_mu1_abs": 0.49046310782432556
|
| 194 |
+
}
|
| 195 |
+
},
|
| 196 |
+
"per_pair": {
|
| 197 |
+
"seed0\u2194seed1": {
|
| 198 |
+
"dcs_signed": 0.8540685176849365,
|
| 199 |
+
"dcs_abs": 0.8541384339332581,
|
| 200 |
+
"frac_sign_flipped": 0.00341796875,
|
| 201 |
+
"alive_source": "stats",
|
| 202 |
+
"frac_denominator": "d_sae",
|
| 203 |
+
"n_alive_a": 12256,
|
| 204 |
+
"n_alive_b": 12250,
|
| 205 |
+
"matched_cos_signed": {
|
| 206 |
+
"0.5": 0.8559465408325195,
|
| 207 |
+
"0.6": 0.8553089499473572,
|
| 208 |
+
"0.7": 0.8564656376838684,
|
| 209 |
+
"0.8": 0.8564656376838684,
|
| 210 |
+
"0.9": 0.8564656376838684
|
| 211 |
+
},
|
| 212 |
+
"unique_frac_signed": {
|
| 213 |
+
"0.5": 0.7342529296875,
|
| 214 |
+
"0.6": 0.9919840494791666,
|
| 215 |
+
"0.7": 0.9971516927083334,
|
| 216 |
+
"0.8": 0.9971516927083334,
|
| 217 |
+
"0.9": 0.9971516927083334
|
| 218 |
+
},
|
| 219 |
+
"dedup_dcs_signed": {
|
| 220 |
+
"0.5": 0.6284812552621588,
|
| 221 |
+
"0.6": 0.8484528357245532,
|
| 222 |
+
"0.7": 0.8540261603629915,
|
| 223 |
+
"0.8": 0.8540261603629915,
|
| 224 |
+
"0.9": 0.8540261603629915
|
| 225 |
+
},
|
| 226 |
+
"matched_frac_signed": {
|
| 227 |
+
"0.5": 0.689208984375,
|
| 228 |
+
"0.6": 0.9131673177083334,
|
| 229 |
+
"0.7": 0.85888671875,
|
| 230 |
+
"0.8": 0.7469889322916666,
|
| 231 |
+
"0.9": 0.548828125
|
| 232 |
+
},
|
| 233 |
+
"matched_cos_abs": {
|
| 234 |
+
"0.5": 0.8558028340339661,
|
| 235 |
+
"0.6": 0.8556124567985535,
|
| 236 |
+
"0.7": 0.8565983176231384,
|
| 237 |
+
"0.8": 0.8565321564674377,
|
| 238 |
+
"0.9": 0.8565024137496948
|
| 239 |
+
},
|
| 240 |
+
"unique_frac_abs": {
|
| 241 |
+
"0.5": 0.7334391276041666,
|
| 242 |
+
"0.6": 0.9891764322916666,
|
| 243 |
+
"0.7": 0.99658203125,
|
| 244 |
+
"0.8": 0.9971110026041666,
|
| 245 |
+
"0.9": 0.9971516927083334
|
| 246 |
+
},
|
| 247 |
+
"dedup_dcs_abs": {
|
| 248 |
+
"0.5": 0.6276792839950455,
|
| 249 |
+
"0.6": 0.8463516774403008,
|
| 250 |
+
"0.7": 0.8536704913422,
|
| 251 |
+
"0.8": 0.8540576372979558,
|
| 252 |
+
"0.9": 0.8540628316792815
|
| 253 |
+
},
|
| 254 |
+
"matched_frac_abs": {
|
| 255 |
+
"0.5": 0.6878255208333334,
|
| 256 |
+
"0.6": 0.9100748697916666,
|
| 257 |
+
"0.7": 0.8588053385416666,
|
| 258 |
+
"0.8": 0.7470703125,
|
| 259 |
+
"0.9": 0.5489908854166666
|
| 260 |
+
}
|
| 261 |
+
},
|
| 262 |
+
"seed0\u2194seed2": {
|
| 263 |
+
"dcs_signed": 0.8531638979911804,
|
| 264 |
+
"dcs_abs": 0.8532828688621521,
|
| 265 |
+
"frac_sign_flipped": 0.004069010416666667,
|
| 266 |
+
"alive_source": "stats",
|
| 267 |
+
"frac_denominator": "d_sae",
|
| 268 |
+
"n_alive_a": 12256,
|
| 269 |
+
"n_alive_b": 12263,
|
| 270 |
+
"matched_cos_signed": {
|
| 271 |
+
"0.5": 0.8537924885749817,
|
| 272 |
+
"0.6": 0.8539842367172241,
|
| 273 |
+
"0.7": 0.8551971912384033,
|
| 274 |
+
"0.8": 0.8551971912384033,
|
| 275 |
+
"0.9": 0.8551971912384033
|
| 276 |
+
},
|
| 277 |
+
"unique_frac_signed": {
|
| 278 |
+
"0.5": 0.7342122395833334,
|
| 279 |
+
"0.6": 0.9921875,
|
| 280 |
+
"0.7": 0.9976806640625,
|
| 281 |
+
"0.8": 0.9976806640625,
|
| 282 |
+
"0.9": 0.9976806640625
|
| 283 |
+
},
|
| 284 |
+
"dedup_dcs_signed": {
|
| 285 |
+
"0.5": 0.6268648951760649,
|
| 286 |
+
"0.6": 0.8473124848678708,
|
| 287 |
+
"0.7": 0.853213701659115,
|
| 288 |
+
"0.8": 0.853213701659115,
|
| 289 |
+
"0.9": 0.853213701659115
|
| 290 |
+
},
|
| 291 |
+
"matched_frac_signed": {
|
| 292 |
+
"0.5": 0.6895345052083334,
|
| 293 |
+
"0.6": 0.914306640625,
|
| 294 |
+
"0.7": 0.8561197916666666,
|
| 295 |
+
"0.8": 0.7405598958333334,
|
| 296 |
+
"0.9": 0.5435384114583334
|
| 297 |
+
},
|
| 298 |
+
"matched_cos_abs": {
|
| 299 |
+
"0.5": 0.8535830974578857,
|
| 300 |
+
"0.6": 0.8543457984924316,
|
| 301 |
+
"0.7": 0.855301558971405,
|
| 302 |
+
"0.8": 0.8552937507629395,
|
| 303 |
+
"0.9": 0.8552937507629395
|
| 304 |
+
},
|
| 305 |
+
"unique_frac_abs": {
|
| 306 |
+
"0.5": 0.7334798177083334,
|
| 307 |
+
"0.6": 0.9893391927083334,
|
| 308 |
+
"0.7": 0.9971516927083334,
|
| 309 |
+
"0.8": 0.9976806640625,
|
| 310 |
+
"0.9": 0.9976806640625
|
| 311 |
+
},
|
| 312 |
+
"dedup_dcs_abs": {
|
| 313 |
+
"0.5": 0.6260859747223246,
|
| 314 |
+
"0.6": 0.8452377825742587,
|
| 315 |
+
"0.7": 0.8528653973044129,
|
| 316 |
+
"0.8": 0.8533100372296758,
|
| 317 |
+
"0.9": 0.8533100372296758
|
| 318 |
+
},
|
| 319 |
+
"matched_frac_abs": {
|
| 320 |
+
"0.5": 0.6883951822916666,
|
| 321 |
+
"0.6": 0.9117838541666666,
|
| 322 |
+
"0.7": 0.8553059895833334,
|
| 323 |
+
"0.8": 0.7406412760416666,
|
| 324 |
+
"0.9": 0.543701171875
|
| 325 |
+
}
|
| 326 |
+
},
|
| 327 |
+
"seed1\u2194seed2": {
|
| 328 |
+
"dcs_signed": 0.854972779750824,
|
| 329 |
+
"dcs_abs": 0.8550770878791809,
|
| 330 |
+
"frac_sign_flipped": 0.00341796875,
|
| 331 |
+
"alive_source": "stats",
|
| 332 |
+
"frac_denominator": "d_sae",
|
| 333 |
+
"n_alive_a": 12250,
|
| 334 |
+
"n_alive_b": 12263,
|
| 335 |
+
"matched_cos_signed": {
|
| 336 |
+
"0.5": 0.8539811968803406,
|
| 337 |
+
"0.6": 0.8557738065719604,
|
| 338 |
+
"0.7": 0.8573965430259705,
|
| 339 |
+
"0.8": 0.8573965430259705,
|
| 340 |
+
"0.9": 0.8573965430259705
|
| 341 |
+
},
|
| 342 |
+
"unique_frac_signed": {
|
| 343 |
+
"0.5": 0.7321370442708334,
|
| 344 |
+
"0.6": 0.9919026692708334,
|
| 345 |
+
"0.7": 0.9974365234375,
|
| 346 |
+
"0.8": 0.9974365234375,
|
| 347 |
+
"0.9": 0.9974365234375
|
| 348 |
+
},
|
| 349 |
+
"dedup_dcs_signed": {
|
| 350 |
+
"0.5": 0.6252312693468411,
|
| 351 |
+
"0.6": 0.8488443230307894,
|
| 352 |
+
"0.7": 0.8551986270831549,
|
| 353 |
+
"0.8": 0.8551986270831549,
|
| 354 |
+
"0.9": 0.8551986270831549
|
| 355 |
+
},
|
| 356 |
+
"matched_frac_signed": {
|
| 357 |
+
"0.5": 0.686767578125,
|
| 358 |
+
"0.6": 0.9159342447916666,
|
| 359 |
+
"0.7": 0.8600260416666666,
|
| 360 |
+
"0.8": 0.74658203125,
|
| 361 |
+
"0.9": 0.5498046875
|
| 362 |
+
},
|
| 363 |
+
"matched_cos_abs": {
|
| 364 |
+
"0.5": 0.854003369808197,
|
| 365 |
+
"0.6": 0.8561245799064636,
|
| 366 |
+
"0.7": 0.8575484752655029,
|
| 367 |
+
"0.8": 0.8574988842010498,
|
| 368 |
+
"0.9": 0.8574696779251099
|
| 369 |
+
},
|
| 370 |
+
"unique_frac_abs": {
|
| 371 |
+
"0.5": 0.7318115234375,
|
| 372 |
+
"0.6": 0.9890950520833334,
|
| 373 |
+
"0.7": 0.996826171875,
|
| 374 |
+
"0.8": 0.9973958333333334,
|
| 375 |
+
"0.9": 0.9974365234375
|
| 376 |
+
},
|
| 377 |
+
"dedup_dcs_abs": {
|
| 378 |
+
"0.5": 0.6249695070800954,
|
| 379 |
+
"0.6": 0.8467885859524056,
|
| 380 |
+
"0.7": 0.8548267637961544,
|
| 381 |
+
"0.8": 0.8552658141901096,
|
| 382 |
+
"0.9": 0.8552715745026944
|
| 383 |
+
},
|
| 384 |
+
"matched_frac_abs": {
|
| 385 |
+
"0.5": 0.6865234375,
|
| 386 |
+
"0.6": 0.91357421875,
|
| 387 |
+
"0.7": 0.8600260416666666,
|
| 388 |
+
"0.8": 0.7462565104166666,
|
| 389 |
+
"0.9": 0.5498046875
|
| 390 |
+
}
|
| 391 |
+
}
|
| 392 |
+
},
|
| 393 |
+
"summary": {
|
| 394 |
+
"n_seeds": 3,
|
| 395 |
+
"n_pairs": 3,
|
| 396 |
+
"report_band": [
|
| 397 |
+
0.6,
|
| 398 |
+
0.7,
|
| 399 |
+
0.8,
|
| 400 |
+
0.9
|
| 401 |
+
],
|
| 402 |
+
"n_dead_mean": 32,
|
| 403 |
+
"n_dead_std": 6.506407098647712,
|
| 404 |
+
"n_dense_mean": 0,
|
| 405 |
+
"n_dense_std": 0.0,
|
| 406 |
+
"l0_mean": 34,
|
| 407 |
+
"l0_std": 0.06916579807077423,
|
| 408 |
+
"recon_r2_mean": 0.8985708450103124,
|
| 409 |
+
"recon_r2_std": 7.01409797481315e-05,
|
| 410 |
+
"l1_per_token_mean": 186.09421918257712,
|
| 411 |
+
"l1_per_token_std": 0.5023468317282951,
|
| 412 |
+
"babel_mu1_signed_mean": 0.48775826891263324,
|
| 413 |
+
"babel_mu1_signed_std": 0.0008159380950787432,
|
| 414 |
+
"n_dup_signed_per_threshold": {
|
| 415 |
+
"0.5": 5925.333333333333,
|
| 416 |
+
"0.6": 130.66666666666666,
|
| 417 |
+
"0.7": 0.0,
|
| 418 |
+
"0.8": 0.0,
|
| 419 |
+
"0.9": 0.0
|
| 420 |
+
},
|
| 421 |
+
"n_dup_signed_band": 32.666666666666664,
|
| 422 |
+
"n_dup_signed_band_std": 2.504163200219453,
|
| 423 |
+
"n_dup_clusters_signed_per_threshold": {
|
| 424 |
+
"0.5": 2682.6666666666665,
|
| 425 |
+
"0.6": 64.33333333333333,
|
| 426 |
+
"0.7": 0.0,
|
| 427 |
+
"0.8": 0.0,
|
| 428 |
+
"0.9": 0.0
|
| 429 |
+
},
|
| 430 |
+
"n_dup_clusters_signed_band": 16.083333333333332,
|
| 431 |
+
"n_dup_clusters_signed_band_std": 1.282900359861721,
|
| 432 |
+
"largest_dup_signed_per_threshold": {
|
| 433 |
+
"0.5": 6.333333333333333,
|
| 434 |
+
"0.6": 3.0,
|
| 435 |
+
"0.7": 1.0,
|
| 436 |
+
"0.8": 1.0,
|
| 437 |
+
"0.9": 1.0
|
| 438 |
+
},
|
| 439 |
+
"largest_dup_signed_band": 1.5,
|
| 440 |
+
"largest_dup_signed_band_std": 0.0,
|
| 441 |
+
"dcs_signed_mean": 0.854068398475647,
|
| 442 |
+
"dcs_signed_std": 0.0009044408857138932,
|
| 443 |
+
"dedup_dcs_signed_per_threshold": {
|
| 444 |
+
"0.5": 0.626859139928355,
|
| 445 |
+
"0.6": 0.8482032145410711,
|
| 446 |
+
"0.7": 0.8541461630350872,
|
| 447 |
+
"0.8": 0.8541461630350872,
|
| 448 |
+
"0.9": 0.8541461630350872
|
| 449 |
+
},
|
| 450 |
+
"dedup_dcs_signed_band": 0.8526604259115832,
|
| 451 |
+
"dedup_dcs_signed_band_std": 0.0009361319306183756,
|
| 452 |
+
"matched_cos_signed_per_threshold": {
|
| 453 |
+
"0.5": 0.8545734087626139,
|
| 454 |
+
"0.6": 0.8550223310788473,
|
| 455 |
+
"0.7": 0.8563531239827474,
|
| 456 |
+
"0.8": 0.8563531239827474,
|
| 457 |
+
"0.9": 0.8563531239827474
|
| 458 |
+
},
|
| 459 |
+
"matched_cos_signed_band": 0.8560204257567724,
|
| 460 |
+
"matched_cos_signed_band_std": 0.0010571259962131921,
|
| 461 |
+
"unique_frac_signed_per_threshold": {
|
| 462 |
+
"0.5": 0.7335340711805557,
|
| 463 |
+
"0.6": 0.9920247395833334,
|
| 464 |
+
"0.7": 0.9974229600694445,
|
| 465 |
+
"0.8": 0.9974229600694445,
|
| 466 |
+
"0.9": 0.9974229600694445
|
| 467 |
+
},
|
| 468 |
+
"unique_frac_signed_band": 0.9960734049479167,
|
| 469 |
+
"unique_frac_signed_band_std": 0.00022448808279124232,
|
| 470 |
+
"matched_frac_signed_per_threshold": {
|
| 471 |
+
"0.5": 0.6885036892361112,
|
| 472 |
+
"0.6": 0.9144694010416666,
|
| 473 |
+
"0.7": 0.8583441840277777,
|
| 474 |
+
"0.8": 0.7447102864583334,
|
| 475 |
+
"0.9": 0.5473904079861112
|
| 476 |
+
},
|
| 477 |
+
"matched_frac_signed_band": 0.7662285698784722,
|
| 478 |
+
"matched_frac_signed_band_std": 0.0023179375372052525,
|
| 479 |
+
"babel_mu1_abs_mean": 0.48954635858535767,
|
| 480 |
+
"babel_mu1_abs_std": 0.0008008541951083467,
|
| 481 |
+
"n_dup_abs_per_threshold": {
|
| 482 |
+
"0.5": 5935.0,
|
| 483 |
+
"0.6": 200.0,
|
| 484 |
+
"0.7": 14.0,
|
| 485 |
+
"0.8": 0.6666666666666666,
|
| 486 |
+
"0.9": 0.0
|
| 487 |
+
},
|
| 488 |
+
"n_dup_abs_band": 53.666666666666664,
|
| 489 |
+
"n_dup_abs_band_std": 2.6496855159307744,
|
| 490 |
+
"n_dup_clusters_abs_per_threshold": {
|
| 491 |
+
"0.5": 2684.6666666666665,
|
| 492 |
+
"0.6": 99.0,
|
| 493 |
+
"0.7": 7.0,
|
| 494 |
+
"0.8": 0.3333333333333333,
|
| 495 |
+
"0.9": 0.0
|
| 496 |
+
},
|
| 497 |
+
"n_dup_clusters_abs_band": 26.583333333333332,
|
| 498 |
+
"n_dup_clusters_abs_band_std": 1.3768926368215255,
|
| 499 |
+
"largest_dup_abs_per_threshold": {
|
| 500 |
+
"0.5": 6.333333333333333,
|
| 501 |
+
"0.6": 3.0,
|
| 502 |
+
"0.7": 2.0,
|
| 503 |
+
"0.8": 1.3333333333333333,
|
| 504 |
+
"0.9": 1.0
|
| 505 |
+
},
|
| 506 |
+
"largest_dup_abs_band": 1.8333333333333333,
|
| 507 |
+
"largest_dup_abs_band_std": 0.14433756729740646,
|
| 508 |
+
"dcs_abs_mean": 0.8541661302248637,
|
| 509 |
+
"dcs_abs_std": 0.0008974300996141616,
|
| 510 |
+
"dedup_dcs_abs_per_threshold": {
|
| 511 |
+
"0.5": 0.6262449219324885,
|
| 512 |
+
"0.6": 0.8461260153223217,
|
| 513 |
+
"0.7": 0.8537875508142557,
|
| 514 |
+
"0.8": 0.8542111629059138,
|
| 515 |
+
"0.9": 0.8542148144705507
|
| 516 |
+
},
|
| 517 |
+
"dedup_dcs_abs_band": 0.8520848858782605,
|
| 518 |
+
"dedup_dcs_abs_band_std": 0.0009296634948064181,
|
| 519 |
+
"matched_cos_abs_per_threshold": {
|
| 520 |
+
"0.5": 0.8544631004333496,
|
| 521 |
+
"0.6": 0.8553609450658163,
|
| 522 |
+
"0.7": 0.8564827839533488,
|
| 523 |
+
"0.8": 0.856441597143809,
|
| 524 |
+
"0.9": 0.856421947479248
|
| 525 |
+
},
|
| 526 |
+
"matched_cos_abs_band": 0.8561768184105555,
|
| 527 |
+
"matched_cos_abs_band_std": 0.0010572823834023232,
|
| 528 |
+
"unique_frac_abs_per_threshold": {
|
| 529 |
+
"0.5": 0.73291015625,
|
| 530 |
+
"0.6": 0.9892035590277778,
|
| 531 |
+
"0.7": 0.9968532986111112,
|
| 532 |
+
"0.8": 0.9973958333333334,
|
| 533 |
+
"0.9": 0.9974229600694445
|
| 534 |
+
},
|
| 535 |
+
"unique_frac_abs_band": 0.9952189127604167,
|
| 536 |
+
"unique_frac_abs_band_std": 0.00023040266220924184,
|
| 537 |
+
"matched_frac_abs_per_threshold": {
|
| 538 |
+
"0.5": 0.6875813802083334,
|
| 539 |
+
"0.6": 0.9118109809027777,
|
| 540 |
+
"0.7": 0.8580457899305555,
|
| 541 |
+
"0.8": 0.744656032986111,
|
| 542 |
+
"0.9": 0.5474989149305555
|
| 543 |
+
},
|
| 544 |
+
"matched_frac_abs_band": 0.7655029296874999,
|
| 545 |
+
"matched_frac_abs_band_std": 0.0023652818407118256,
|
| 546 |
+
"alive_source": "stats",
|
| 547 |
+
"frac_denominator": "d_sae",
|
| 548 |
+
"d_sae": 12288
|
| 549 |
+
}
|
| 550 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 16,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 0,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 12.5,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 30,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_samples": 20000,
|
| 3 |
+
"n_positions": 20480000,
|
| 4 |
+
"l0_per_token_0": 34.09826708984375,
|
| 5 |
+
"l0_per_token_0.001": 34.098253857421874,
|
| 6 |
+
"l0_per_token_0.01": 34.09807836914062,
|
| 7 |
+
"l0_per_token_0.05": 34.09750639648438,
|
| 8 |
+
"l0_per_token_0.1": 34.096228662109375,
|
| 9 |
+
"l0_per_token_0.5": 34.0711404296875,
|
| 10 |
+
"l0_per_token_1.0": 34.01213842773438,
|
| 11 |
+
"l1_per_token": 186.09178804915646,
|
| 12 |
+
"r2_score": 0.8984899544332474,
|
| 13 |
+
"l2_loss": 2.3806576455658055
|
| 14 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "fineweb",
|
| 3 |
+
"split": "train",
|
| 4 |
+
"hf_dataset": "HuggingFaceFW/fineweb",
|
| 5 |
+
"hf_dataset_config": null,
|
| 6 |
+
"hf_dataset_text_field": "text",
|
| 7 |
+
"context_length": 1024,
|
| 8 |
+
"num_samples_requested": 20000,
|
| 9 |
+
"num_samples_actual": 20000,
|
| 10 |
+
"batch_seqs": 32,
|
| 11 |
+
"stats_top_k": 100,
|
| 12 |
+
"stream_seed": 0,
|
| 13 |
+
"sparsity_thresholds": [
|
| 14 |
+
0.001,
|
| 15 |
+
0.01,
|
| 16 |
+
0.05,
|
| 17 |
+
0.1,
|
| 18 |
+
0.5,
|
| 19 |
+
1.0
|
| 20 |
+
],
|
| 21 |
+
"model_hf_name": "openai-community/gpt2",
|
| 22 |
+
"hook_target_name": "transformer.h.8",
|
| 23 |
+
"tokenizer_hf_name": "openai-community/gpt2",
|
| 24 |
+
"d_sae": 12288,
|
| 25 |
+
"d_in": 768
|
| 26 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 16,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 1,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 12.5,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 30,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_samples": 20000,
|
| 3 |
+
"n_positions": 20480000,
|
| 4 |
+
"l0_per_token_0": 34.087190966796875,
|
| 5 |
+
"l0_per_token_0.001": 34.0871806640625,
|
| 6 |
+
"l0_per_token_0.01": 34.087062158203125,
|
| 7 |
+
"l0_per_token_0.05": 34.086601708984375,
|
| 8 |
+
"l0_per_token_0.1": 34.08566435546875,
|
| 9 |
+
"l0_per_token_0.5": 34.06172890625,
|
| 10 |
+
"l0_per_token_1.0": 34.00160493164063,
|
| 11 |
+
"l1_per_token": 185.59309232967692,
|
| 12 |
+
"r2_score": 0.8986077841689825,
|
| 13 |
+
"l2_loss": 2.377894251463415
|
| 14 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_32/seed1/stats.fineweb.split_train.seed_no/meta.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "fineweb",
|
| 3 |
+
"split": "train",
|
| 4 |
+
"hf_dataset": "HuggingFaceFW/fineweb",
|
| 5 |
+
"hf_dataset_config": null,
|
| 6 |
+
"hf_dataset_text_field": "text",
|
| 7 |
+
"context_length": 1024,
|
| 8 |
+
"num_samples_requested": 20000,
|
| 9 |
+
"num_samples_actual": 20000,
|
| 10 |
+
"batch_seqs": 32,
|
| 11 |
+
"stats_top_k": 100,
|
| 12 |
+
"stream_seed": 0,
|
| 13 |
+
"sparsity_thresholds": [
|
| 14 |
+
0.001,
|
| 15 |
+
0.01,
|
| 16 |
+
0.05,
|
| 17 |
+
0.1,
|
| 18 |
+
0.5,
|
| 19 |
+
1.0
|
| 20 |
+
],
|
| 21 |
+
"model_hf_name": "openai-community/gpt2",
|
| 22 |
+
"hook_target_name": "transformer.h.8",
|
| 23 |
+
"tokenizer_hf_name": "openai-community/gpt2",
|
| 24 |
+
"d_sae": 12288,
|
| 25 |
+
"d_in": 768
|
| 26 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 16,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 2,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 12.5,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 30,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_samples": 20000,
|
| 3 |
+
"n_positions": 20480000,
|
| 4 |
+
"l0_per_token_0": 34.21214306640625,
|
| 5 |
+
"l0_per_token_0.001": 34.21213720703125,
|
| 6 |
+
"l0_per_token_0.01": 34.21210283203125,
|
| 7 |
+
"l0_per_token_0.05": 34.21181171875,
|
| 8 |
+
"l0_per_token_0.1": 34.21116733398438,
|
| 9 |
+
"l0_per_token_0.5": 34.19097841796875,
|
| 10 |
+
"l0_per_token_1.0": 34.13543442382812,
|
| 11 |
+
"l1_per_token": 186.597777168898,
|
| 12 |
+
"r2_score": 0.898614796428707,
|
| 13 |
+
"l2_loss": 2.3777297969049256
|
| 14 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_32/seed2/stats.fineweb.split_train.seed_no/meta.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "fineweb",
|
| 3 |
+
"split": "train",
|
| 4 |
+
"hf_dataset": "HuggingFaceFW/fineweb",
|
| 5 |
+
"hf_dataset_config": null,
|
| 6 |
+
"hf_dataset_text_field": "text",
|
| 7 |
+
"context_length": 1024,
|
| 8 |
+
"num_samples_requested": 20000,
|
| 9 |
+
"num_samples_actual": 20000,
|
| 10 |
+
"batch_seqs": 32,
|
| 11 |
+
"stats_top_k": 100,
|
| 12 |
+
"stream_seed": 0,
|
| 13 |
+
"sparsity_thresholds": [
|
| 14 |
+
0.001,
|
| 15 |
+
0.01,
|
| 16 |
+
0.05,
|
| 17 |
+
0.1,
|
| 18 |
+
0.5,
|
| 19 |
+
1.0
|
| 20 |
+
],
|
| 21 |
+
"model_hf_name": "openai-community/gpt2",
|
| 22 |
+
"hook_target_name": "transformer.h.8",
|
| 23 |
+
"tokenizer_hf_name": "openai-community/gpt2",
|
| 24 |
+
"d_sae": 12288,
|
| 25 |
+
"d_in": 768
|
| 26 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_64/code_metrics_step_final.json
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": 1,
|
| 3 |
+
"step": "final",
|
| 4 |
+
"n_tokens": 600000,
|
| 5 |
+
"thresholds": [
|
| 6 |
+
0.5,
|
| 7 |
+
0.6,
|
| 8 |
+
0.7,
|
| 9 |
+
0.8,
|
| 10 |
+
0.9
|
| 11 |
+
],
|
| 12 |
+
"seeds": [
|
| 13 |
+
"seed0",
|
| 14 |
+
"seed1",
|
| 15 |
+
"seed2"
|
| 16 |
+
],
|
| 17 |
+
"per_seed": {
|
| 18 |
+
"seed0": {
|
| 19 |
+
"l0": 68.33619833333333,
|
| 20 |
+
"recon_r2": 0.9322517340770707,
|
| 21 |
+
"l1_per_token": 255.8558036472112
|
| 22 |
+
},
|
| 23 |
+
"seed1": {
|
| 24 |
+
"l0": 68.25313666666666,
|
| 25 |
+
"recon_r2": 0.9323407862864682,
|
| 26 |
+
"l1_per_token": 255.34123201401795
|
| 27 |
+
},
|
| 28 |
+
"seed2": {
|
| 29 |
+
"l0": 68.50301666666667,
|
| 30 |
+
"recon_r2": 0.9322924700450567,
|
| 31 |
+
"l1_per_token": 256.96076816634786
|
| 32 |
+
}
|
| 33 |
+
},
|
| 34 |
+
"per_pair": {
|
| 35 |
+
"seed0\u2194seed1": {
|
| 36 |
+
"alive_source": "stats",
|
| 37 |
+
"mcc_signed": 0.7768021305413016,
|
| 38 |
+
"mcc_abs": 0.7768030977677247,
|
| 39 |
+
"act_iou": 0.46257098043952416,
|
| 40 |
+
"act_iou_n_pairs": 12288,
|
| 41 |
+
"input_r2": 0.9875357167160536,
|
| 42 |
+
"matched_mcc_signed": {
|
| 43 |
+
"0.6": 0.8336217505790626,
|
| 44 |
+
"0.7": 0.8617261810456283,
|
| 45 |
+
"0.8": 0.8989884177254652,
|
| 46 |
+
"0.9": 0.9414152728089523
|
| 47 |
+
},
|
| 48 |
+
"matched_mcc_abs": {
|
| 49 |
+
"0.6": 0.8338279584249467,
|
| 50 |
+
"0.7": 0.8612219114060355,
|
| 51 |
+
"0.8": 0.898788023707797,
|
| 52 |
+
"0.9": 0.9413730426214577
|
| 53 |
+
}
|
| 54 |
+
},
|
| 55 |
+
"seed0\u2194seed2": {
|
| 56 |
+
"alive_source": "stats",
|
| 57 |
+
"mcc_signed": 0.775137631695522,
|
| 58 |
+
"mcc_abs": 0.7751397889782531,
|
| 59 |
+
"act_iou": 0.46259933192133634,
|
| 60 |
+
"act_iou_n_pairs": 12288,
|
| 61 |
+
"input_r2": 0.9876135734799633,
|
| 62 |
+
"matched_mcc_signed": {
|
| 63 |
+
"0.6": 0.8327112898532983,
|
| 64 |
+
"0.7": 0.8610406097487728,
|
| 65 |
+
"0.8": 0.8985328214583472,
|
| 66 |
+
"0.9": 0.9411809905755064
|
| 67 |
+
},
|
| 68 |
+
"matched_mcc_abs": {
|
| 69 |
+
"0.6": 0.8327528887150836,
|
| 70 |
+
"0.7": 0.8610792045865284,
|
| 71 |
+
"0.8": 0.8986460702756782,
|
| 72 |
+
"0.9": 0.9411762675459137
|
| 73 |
+
}
|
| 74 |
+
},
|
| 75 |
+
"seed1\u2194seed2": {
|
| 76 |
+
"alive_source": "stats",
|
| 77 |
+
"mcc_signed": 0.775594728791976,
|
| 78 |
+
"mcc_abs": 0.7755953713512026,
|
| 79 |
+
"act_iou": 0.46374883008495243,
|
| 80 |
+
"act_iou_n_pairs": 12288,
|
| 81 |
+
"input_r2": 0.987641363010334,
|
| 82 |
+
"matched_mcc_signed": {
|
| 83 |
+
"0.6": 0.833073454668621,
|
| 84 |
+
"0.7": 0.8626854635967414,
|
| 85 |
+
"0.8": 0.9002780058304956,
|
| 86 |
+
"0.9": 0.9413255147035597
|
| 87 |
+
},
|
| 88 |
+
"matched_mcc_abs": {
|
| 89 |
+
"0.6": 0.8339831061502786,
|
| 90 |
+
"0.7": 0.8627077226289882,
|
| 91 |
+
"0.8": 0.8999366437006253,
|
| 92 |
+
"0.9": 0.9412923738555892
|
| 93 |
+
}
|
| 94 |
+
}
|
| 95 |
+
},
|
| 96 |
+
"summary": {
|
| 97 |
+
"n_seeds": 3,
|
| 98 |
+
"n_pairs": 3,
|
| 99 |
+
"report_band": [
|
| 100 |
+
0.6,
|
| 101 |
+
0.7,
|
| 102 |
+
0.8,
|
| 103 |
+
0.9
|
| 104 |
+
],
|
| 105 |
+
"l0_mean": 68.36411722222222,
|
| 106 |
+
"l0_std": 0.12725801297990336,
|
| 107 |
+
"recon_r2_mean": 0.9322949968028652,
|
| 108 |
+
"recon_r2_std": 4.457984273649158e-05,
|
| 109 |
+
"l1_per_token_mean": 256.052601275859,
|
| 110 |
+
"l1_per_token_std": 0.8275091039722443,
|
| 111 |
+
"input_r2_mean": 0.9875968844021169,
|
| 112 |
+
"input_r2_std": 5.4764759319457435e-05,
|
| 113 |
+
"mcc_signed_mean": 0.7758448303429333,
|
| 114 |
+
"mcc_signed_std": 0.0008599722037623513,
|
| 115 |
+
"mcc_abs_mean": 0.7758460860323936,
|
| 116 |
+
"mcc_abs_std": 0.0008595303489737128,
|
| 117 |
+
"matched_mcc_signed_per_threshold": {
|
| 118 |
+
"0.6": 0.833135498366994,
|
| 119 |
+
"0.7": 0.8618174181303808,
|
| 120 |
+
"0.8": 0.8992664150047694,
|
| 121 |
+
"0.9": 0.9413072593626728
|
| 122 |
+
},
|
| 123 |
+
"matched_mcc_signed_band": 0.8838816477162043,
|
| 124 |
+
"matched_mcc_abs_per_threshold": {
|
| 125 |
+
"0.6": 0.8335213177634363,
|
| 126 |
+
"0.7": 0.8616696128738507,
|
| 127 |
+
"0.8": 0.8991235792280335,
|
| 128 |
+
"0.9": 0.9412805613409869
|
| 129 |
+
},
|
| 130 |
+
"matched_mcc_abs_band": 0.8838987678015768,
|
| 131 |
+
"act_iou_mean": 0.4629730474819376,
|
| 132 |
+
"act_iou_std": 0.0006719969769169633,
|
| 133 |
+
"act_iou_n_pairs": 12288.0,
|
| 134 |
+
"alive_source": "stats"
|
| 135 |
+
}
|
| 136 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_64/dict_metrics_step_final.json
ADDED
|
@@ -0,0 +1,550 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": 2,
|
| 3 |
+
"step": "final",
|
| 4 |
+
"linkage": "complete",
|
| 5 |
+
"thresholds": [
|
| 6 |
+
0.5,
|
| 7 |
+
0.6,
|
| 8 |
+
0.7,
|
| 9 |
+
0.8,
|
| 10 |
+
0.9
|
| 11 |
+
],
|
| 12 |
+
"seeds": [
|
| 13 |
+
"seed0",
|
| 14 |
+
"seed1",
|
| 15 |
+
"seed2"
|
| 16 |
+
],
|
| 17 |
+
"per_seed": {
|
| 18 |
+
"seed0": {
|
| 19 |
+
"d_sae": 12288,
|
| 20 |
+
"d_in": 768,
|
| 21 |
+
"l0": 68,
|
| 22 |
+
"l0_raw": 68.38207143554688,
|
| 23 |
+
"recon_r2": 0.9321300780911635,
|
| 24 |
+
"l1_per_token": 255.81543037355635,
|
| 25 |
+
"stats_source": "stats",
|
| 26 |
+
"n_dead": 0,
|
| 27 |
+
"dead_rate": 0.0,
|
| 28 |
+
"alive_source": "stats",
|
| 29 |
+
"n_dense": 14,
|
| 30 |
+
"dense_rate": 0.0011393229166666667,
|
| 31 |
+
"dense_threshold": 0.1,
|
| 32 |
+
"n_dup_signed": {
|
| 33 |
+
"0.5": 3315,
|
| 34 |
+
"0.6": 2,
|
| 35 |
+
"0.7": 0,
|
| 36 |
+
"0.8": 0,
|
| 37 |
+
"0.9": 0
|
| 38 |
+
},
|
| 39 |
+
"n_dup_clusters_signed": {
|
| 40 |
+
"0.5": 1541,
|
| 41 |
+
"0.6": 1,
|
| 42 |
+
"0.7": 0,
|
| 43 |
+
"0.8": 0,
|
| 44 |
+
"0.9": 0
|
| 45 |
+
},
|
| 46 |
+
"largest_dup_signed": {
|
| 47 |
+
"0.5": 5,
|
| 48 |
+
"0.6": 2,
|
| 49 |
+
"0.7": 1,
|
| 50 |
+
"0.8": 1,
|
| 51 |
+
"0.9": 1
|
| 52 |
+
},
|
| 53 |
+
"babel_mu1_signed": 0.4383678436279297,
|
| 54 |
+
"n_dup_abs": {
|
| 55 |
+
"0.5": 3347,
|
| 56 |
+
"0.6": 110,
|
| 57 |
+
"0.7": 30,
|
| 58 |
+
"0.8": 2,
|
| 59 |
+
"0.9": 0
|
| 60 |
+
},
|
| 61 |
+
"n_dup_clusters_abs": {
|
| 62 |
+
"0.5": 1557,
|
| 63 |
+
"0.6": 55,
|
| 64 |
+
"0.7": 15,
|
| 65 |
+
"0.8": 1,
|
| 66 |
+
"0.9": 0
|
| 67 |
+
},
|
| 68 |
+
"largest_dup_abs": {
|
| 69 |
+
"0.5": 5,
|
| 70 |
+
"0.6": 2,
|
| 71 |
+
"0.7": 2,
|
| 72 |
+
"0.8": 2,
|
| 73 |
+
"0.9": 1
|
| 74 |
+
},
|
| 75 |
+
"babel_mu1_abs": 0.4412466585636139
|
| 76 |
+
},
|
| 77 |
+
"seed1": {
|
| 78 |
+
"d_sae": 12288,
|
| 79 |
+
"d_in": 768,
|
| 80 |
+
"l0": 68,
|
| 81 |
+
"l0_raw": 68.27429501953125,
|
| 82 |
+
"recon_r2": 0.9322215810946727,
|
| 83 |
+
"l1_per_token": 255.2836143987286,
|
| 84 |
+
"stats_source": "stats",
|
| 85 |
+
"n_dead": 0,
|
| 86 |
+
"dead_rate": 0.0,
|
| 87 |
+
"alive_source": "stats",
|
| 88 |
+
"n_dense": 16,
|
| 89 |
+
"dense_rate": 0.0013020833333333333,
|
| 90 |
+
"dense_threshold": 0.1,
|
| 91 |
+
"n_dup_signed": {
|
| 92 |
+
"0.5": 3240,
|
| 93 |
+
"0.6": 2,
|
| 94 |
+
"0.7": 0,
|
| 95 |
+
"0.8": 0,
|
| 96 |
+
"0.9": 0
|
| 97 |
+
},
|
| 98 |
+
"n_dup_clusters_signed": {
|
| 99 |
+
"0.5": 1500,
|
| 100 |
+
"0.6": 1,
|
| 101 |
+
"0.7": 0,
|
| 102 |
+
"0.8": 0,
|
| 103 |
+
"0.9": 0
|
| 104 |
+
},
|
| 105 |
+
"largest_dup_signed": {
|
| 106 |
+
"0.5": 7,
|
| 107 |
+
"0.6": 2,
|
| 108 |
+
"0.7": 1,
|
| 109 |
+
"0.8": 1,
|
| 110 |
+
"0.9": 1
|
| 111 |
+
},
|
| 112 |
+
"babel_mu1_signed": 0.4383070468902588,
|
| 113 |
+
"n_dup_abs": {
|
| 114 |
+
"0.5": 3280,
|
| 115 |
+
"0.6": 98,
|
| 116 |
+
"0.7": 18,
|
| 117 |
+
"0.8": 0,
|
| 118 |
+
"0.9": 0
|
| 119 |
+
},
|
| 120 |
+
"n_dup_clusters_abs": {
|
| 121 |
+
"0.5": 1518,
|
| 122 |
+
"0.6": 49,
|
| 123 |
+
"0.7": 9,
|
| 124 |
+
"0.8": 0,
|
| 125 |
+
"0.9": 0
|
| 126 |
+
},
|
| 127 |
+
"largest_dup_abs": {
|
| 128 |
+
"0.5": 7,
|
| 129 |
+
"0.6": 2,
|
| 130 |
+
"0.7": 2,
|
| 131 |
+
"0.8": 1,
|
| 132 |
+
"0.9": 1
|
| 133 |
+
},
|
| 134 |
+
"babel_mu1_abs": 0.44103512167930603
|
| 135 |
+
},
|
| 136 |
+
"seed2": {
|
| 137 |
+
"d_sae": 12288,
|
| 138 |
+
"d_in": 768,
|
| 139 |
+
"l0": 69,
|
| 140 |
+
"l0_raw": 68.547587109375,
|
| 141 |
+
"recon_r2": 0.9321723959031226,
|
| 142 |
+
"l1_per_token": 256.91514224984536,
|
| 143 |
+
"stats_source": "stats",
|
| 144 |
+
"n_dead": 0,
|
| 145 |
+
"dead_rate": 0.0,
|
| 146 |
+
"alive_source": "stats",
|
| 147 |
+
"n_dense": 13,
|
| 148 |
+
"dense_rate": 0.0010579427083333333,
|
| 149 |
+
"dense_threshold": 0.1,
|
| 150 |
+
"n_dup_signed": {
|
| 151 |
+
"0.5": 3275,
|
| 152 |
+
"0.6": 2,
|
| 153 |
+
"0.7": 0,
|
| 154 |
+
"0.8": 0,
|
| 155 |
+
"0.9": 0
|
| 156 |
+
},
|
| 157 |
+
"n_dup_clusters_signed": {
|
| 158 |
+
"0.5": 1520,
|
| 159 |
+
"0.6": 1,
|
| 160 |
+
"0.7": 0,
|
| 161 |
+
"0.8": 0,
|
| 162 |
+
"0.9": 0
|
| 163 |
+
},
|
| 164 |
+
"largest_dup_signed": {
|
| 165 |
+
"0.5": 6,
|
| 166 |
+
"0.6": 2,
|
| 167 |
+
"0.7": 1,
|
| 168 |
+
"0.8": 1,
|
| 169 |
+
"0.9": 1
|
| 170 |
+
},
|
| 171 |
+
"babel_mu1_signed": 0.438535213470459,
|
| 172 |
+
"n_dup_abs": {
|
| 173 |
+
"0.5": 3318,
|
| 174 |
+
"0.6": 98,
|
| 175 |
+
"0.7": 16,
|
| 176 |
+
"0.8": 0,
|
| 177 |
+
"0.9": 0
|
| 178 |
+
},
|
| 179 |
+
"n_dup_clusters_abs": {
|
| 180 |
+
"0.5": 1541,
|
| 181 |
+
"0.6": 49,
|
| 182 |
+
"0.7": 8,
|
| 183 |
+
"0.8": 0,
|
| 184 |
+
"0.9": 0
|
| 185 |
+
},
|
| 186 |
+
"largest_dup_abs": {
|
| 187 |
+
"0.5": 6,
|
| 188 |
+
"0.6": 2,
|
| 189 |
+
"0.7": 2,
|
| 190 |
+
"0.8": 1,
|
| 191 |
+
"0.9": 1
|
| 192 |
+
},
|
| 193 |
+
"babel_mu1_abs": 0.4413144290447235
|
| 194 |
+
}
|
| 195 |
+
},
|
| 196 |
+
"per_pair": {
|
| 197 |
+
"seed0\u2194seed1": {
|
| 198 |
+
"dcs_signed": 0.842194139957428,
|
| 199 |
+
"dcs_abs": 0.8423486351966858,
|
| 200 |
+
"frac_sign_flipped": 0.0035807291666666665,
|
| 201 |
+
"alive_source": "stats",
|
| 202 |
+
"frac_denominator": "d_sae",
|
| 203 |
+
"n_alive_a": 12288,
|
| 204 |
+
"n_alive_b": 12288,
|
| 205 |
+
"matched_cos_signed": {
|
| 206 |
+
"0.5": 0.8376299738883972,
|
| 207 |
+
"0.6": 0.8421900868415833,
|
| 208 |
+
"0.7": 0.842194139957428,
|
| 209 |
+
"0.8": 0.842194139957428,
|
| 210 |
+
"0.9": 0.842194139957428
|
| 211 |
+
},
|
| 212 |
+
"unique_frac_signed": {
|
| 213 |
+
"0.5": 0.8570149739583334,
|
| 214 |
+
"0.6": 0.9999186197916666,
|
| 215 |
+
"0.7": 1.0,
|
| 216 |
+
"0.8": 1.0,
|
| 217 |
+
"0.9": 1.0
|
| 218 |
+
},
|
| 219 |
+
"dedup_dcs_signed": {
|
| 220 |
+
"0.5": 0.7178614302586842,
|
| 221 |
+
"0.6": 0.8421215492368598,
|
| 222 |
+
"0.7": 0.842194139957428,
|
| 223 |
+
"0.8": 0.842194139957428,
|
| 224 |
+
"0.9": 0.842194139957428
|
| 225 |
+
},
|
| 226 |
+
"matched_frac_signed": {
|
| 227 |
+
"0.5": 0.7935384114583334,
|
| 228 |
+
"0.6": 0.903076171875,
|
| 229 |
+
"0.7": 0.835205078125,
|
| 230 |
+
"0.8": 0.7178548177083334,
|
| 231 |
+
"0.9": 0.5221354166666666
|
| 232 |
+
},
|
| 233 |
+
"matched_cos_abs": {
|
| 234 |
+
"0.5": 0.8379883766174316,
|
| 235 |
+
"0.6": 0.8430367708206177,
|
| 236 |
+
"0.7": 0.8426423668861389,
|
| 237 |
+
"0.8": 0.8423916697502136,
|
| 238 |
+
"0.9": 0.8423486351966858
|
| 239 |
+
},
|
| 240 |
+
"unique_frac_abs": {
|
| 241 |
+
"0.5": 0.85546875,
|
| 242 |
+
"0.6": 0.9957682291666666,
|
| 243 |
+
"0.7": 0.9990234375,
|
| 244 |
+
"0.8": 0.9999593098958334,
|
| 245 |
+
"0.9": 1.0
|
| 246 |
+
},
|
| 247 |
+
"dedup_dcs_abs": {
|
| 248 |
+
"0.5": 0.7168728690594435,
|
| 249 |
+
"0.6": 0.8394692324024314,
|
| 250 |
+
"0.7": 0.8418194739497267,
|
| 251 |
+
"0.8": 0.8423573927454224,
|
| 252 |
+
"0.9": 0.8423486351966858
|
| 253 |
+
},
|
| 254 |
+
"matched_frac_abs": {
|
| 255 |
+
"0.5": 0.7913411458333334,
|
| 256 |
+
"0.6": 0.8990885416666666,
|
| 257 |
+
"0.7": 0.83447265625,
|
| 258 |
+
"0.8": 0.71826171875,
|
| 259 |
+
"0.9": 0.5224609375
|
| 260 |
+
}
|
| 261 |
+
},
|
| 262 |
+
"seed0\u2194seed2": {
|
| 263 |
+
"dcs_signed": 0.8415024280548096,
|
| 264 |
+
"dcs_abs": 0.8416246771812439,
|
| 265 |
+
"frac_sign_flipped": 0.0033365885416666665,
|
| 266 |
+
"alive_source": "stats",
|
| 267 |
+
"frac_denominator": "d_sae",
|
| 268 |
+
"n_alive_a": 12288,
|
| 269 |
+
"n_alive_b": 12288,
|
| 270 |
+
"matched_cos_signed": {
|
| 271 |
+
"0.5": 0.8373281359672546,
|
| 272 |
+
"0.6": 0.8414965867996216,
|
| 273 |
+
"0.7": 0.8415024280548096,
|
| 274 |
+
"0.8": 0.8415024280548096,
|
| 275 |
+
"0.9": 0.8415024280548096
|
| 276 |
+
},
|
| 277 |
+
"unique_frac_signed": {
|
| 278 |
+
"0.5": 0.8564046223958334,
|
| 279 |
+
"0.6": 0.9999186197916666,
|
| 280 |
+
"0.7": 1.0,
|
| 281 |
+
"0.8": 1.0,
|
| 282 |
+
"0.9": 1.0
|
| 283 |
+
},
|
| 284 |
+
"dedup_dcs_signed": {
|
| 285 |
+
"0.5": 0.7170916861044437,
|
| 286 |
+
"0.6": 0.841428105632076,
|
| 287 |
+
"0.7": 0.8415024280548096,
|
| 288 |
+
"0.8": 0.8415024280548096,
|
| 289 |
+
"0.9": 0.8415024280548096
|
| 290 |
+
},
|
| 291 |
+
"matched_frac_signed": {
|
| 292 |
+
"0.5": 0.7937825520833334,
|
| 293 |
+
"0.6": 0.9007161458333334,
|
| 294 |
+
"0.7": 0.8323567708333334,
|
| 295 |
+
"0.8": 0.7145182291666666,
|
| 296 |
+
"0.9": 0.52197265625
|
| 297 |
+
},
|
| 298 |
+
"matched_cos_abs": {
|
| 299 |
+
"0.5": 0.8376181721687317,
|
| 300 |
+
"0.6": 0.8422266244888306,
|
| 301 |
+
"0.7": 0.8418663144111633,
|
| 302 |
+
"0.8": 0.8416663408279419,
|
| 303 |
+
"0.9": 0.8416246771812439
|
| 304 |
+
},
|
| 305 |
+
"unique_frac_abs": {
|
| 306 |
+
"0.5": 0.8548583984375,
|
| 307 |
+
"0.6": 0.9957682291666666,
|
| 308 |
+
"0.7": 0.9990641276041666,
|
| 309 |
+
"0.8": 0.9999593098958334,
|
| 310 |
+
"0.9": 1.0
|
| 311 |
+
},
|
| 312 |
+
"dedup_dcs_abs": {
|
| 313 |
+
"0.5": 0.7160449291623081,
|
| 314 |
+
"0.6": 0.8386625144242619,
|
| 315 |
+
"0.7": 0.8410784349665239,
|
| 316 |
+
"0.8": 0.8416320933368601,
|
| 317 |
+
"0.9": 0.8416246771812439
|
| 318 |
+
},
|
| 319 |
+
"matched_frac_abs": {
|
| 320 |
+
"0.5": 0.79296875,
|
| 321 |
+
"0.6": 0.8961588541666666,
|
| 322 |
+
"0.7": 0.8316243489583334,
|
| 323 |
+
"0.8": 0.714599609375,
|
| 324 |
+
"0.9": 0.5220540364583334
|
| 325 |
+
}
|
| 326 |
+
},
|
| 327 |
+
"seed1\u2194seed2": {
|
| 328 |
+
"dcs_signed": 0.8420475125312805,
|
| 329 |
+
"dcs_abs": 0.8422133326530457,
|
| 330 |
+
"frac_sign_flipped": 0.004150390625,
|
| 331 |
+
"alive_source": "stats",
|
| 332 |
+
"frac_denominator": "d_sae",
|
| 333 |
+
"n_alive_a": 12288,
|
| 334 |
+
"n_alive_b": 12288,
|
| 335 |
+
"matched_cos_signed": {
|
| 336 |
+
"0.5": 0.8369420766830444,
|
| 337 |
+
"0.6": 0.8420504331588745,
|
| 338 |
+
"0.7": 0.8420475125312805,
|
| 339 |
+
"0.8": 0.8420475125312805,
|
| 340 |
+
"0.9": 0.8420475125312805
|
| 341 |
+
},
|
| 342 |
+
"unique_frac_signed": {
|
| 343 |
+
"0.5": 0.8577880859375,
|
| 344 |
+
"0.6": 0.9999186197916666,
|
| 345 |
+
"0.7": 1.0,
|
| 346 |
+
"0.8": 1.0,
|
| 347 |
+
"0.9": 1.0
|
| 348 |
+
},
|
| 349 |
+
"dedup_dcs_signed": {
|
| 350 |
+
"0.5": 0.717918941998505,
|
| 351 |
+
"0.6": 0.8419819069191968,
|
| 352 |
+
"0.7": 0.8420475125312805,
|
| 353 |
+
"0.8": 0.8420475125312805,
|
| 354 |
+
"0.9": 0.8420475125312805
|
| 355 |
+
},
|
| 356 |
+
"matched_frac_signed": {
|
| 357 |
+
"0.5": 0.7963053385416666,
|
| 358 |
+
"0.6": 0.9016927083333334,
|
| 359 |
+
"0.7": 0.8310546875,
|
| 360 |
+
"0.8": 0.7145182291666666,
|
| 361 |
+
"0.9": 0.526611328125
|
| 362 |
+
},
|
| 363 |
+
"matched_cos_abs": {
|
| 364 |
+
"0.5": 0.8375152945518494,
|
| 365 |
+
"0.6": 0.8427015542984009,
|
| 366 |
+
"0.7": 0.8422761559486389,
|
| 367 |
+
"0.8": 0.8422133326530457,
|
| 368 |
+
"0.9": 0.8422133326530457
|
| 369 |
+
},
|
| 370 |
+
"unique_frac_abs": {
|
| 371 |
+
"0.5": 0.8559977213541666,
|
| 372 |
+
"0.6": 0.9960123697916666,
|
| 373 |
+
"0.7": 0.9993082682291666,
|
| 374 |
+
"0.8": 1.0,
|
| 375 |
+
"0.9": 1.0
|
| 376 |
+
},
|
| 377 |
+
"dedup_dcs_abs": {
|
| 378 |
+
"0.5": 0.7169111837356468,
|
| 379 |
+
"0.6": 0.839341172123871,
|
| 380 |
+
"0.7": 0.8416935267717538,
|
| 381 |
+
"0.8": 0.8422133326530457,
|
| 382 |
+
"0.9": 0.8422133326530457
|
| 383 |
+
},
|
| 384 |
+
"matched_frac_abs": {
|
| 385 |
+
"0.5": 0.794921875,
|
| 386 |
+
"0.6": 0.8975423177083334,
|
| 387 |
+
"0.7": 0.8302408854166666,
|
| 388 |
+
"0.8": 0.7154134114583334,
|
| 389 |
+
"0.9": 0.5266927083333334
|
| 390 |
+
}
|
| 391 |
+
}
|
| 392 |
+
},
|
| 393 |
+
"summary": {
|
| 394 |
+
"n_seeds": 3,
|
| 395 |
+
"n_pairs": 3,
|
| 396 |
+
"report_band": [
|
| 397 |
+
0.6,
|
| 398 |
+
0.7,
|
| 399 |
+
0.8,
|
| 400 |
+
0.9
|
| 401 |
+
],
|
| 402 |
+
"n_dead_mean": 0,
|
| 403 |
+
"n_dead_std": 0.0,
|
| 404 |
+
"n_dense_mean": 14,
|
| 405 |
+
"n_dense_std": 1.5275252316519468,
|
| 406 |
+
"l0_mean": 68,
|
| 407 |
+
"l0_std": 0.13766530314758454,
|
| 408 |
+
"recon_r2_mean": 0.932174685029653,
|
| 409 |
+
"recon_r2_std": 3.464368346113687e-05,
|
| 410 |
+
"l1_per_token_mean": 256.00472900737674,
|
| 411 |
+
"l1_per_token_std": 0.8320763648757366,
|
| 412 |
+
"babel_mu1_signed_mean": 0.4384033679962158,
|
| 413 |
+
"babel_mu1_signed_std": 0.00011815871798838899,
|
| 414 |
+
"n_dup_signed_per_threshold": {
|
| 415 |
+
"0.5": 3276.6666666666665,
|
| 416 |
+
"0.6": 2.0,
|
| 417 |
+
"0.7": 0.0,
|
| 418 |
+
"0.8": 0.0,
|
| 419 |
+
"0.9": 0.0
|
| 420 |
+
},
|
| 421 |
+
"n_dup_signed_band": 0.5,
|
| 422 |
+
"n_dup_signed_band_std": 0.0,
|
| 423 |
+
"n_dup_clusters_signed_per_threshold": {
|
| 424 |
+
"0.5": 1520.3333333333333,
|
| 425 |
+
"0.6": 1.0,
|
| 426 |
+
"0.7": 0.0,
|
| 427 |
+
"0.8": 0.0,
|
| 428 |
+
"0.9": 0.0
|
| 429 |
+
},
|
| 430 |
+
"n_dup_clusters_signed_band": 0.25,
|
| 431 |
+
"n_dup_clusters_signed_band_std": 0.0,
|
| 432 |
+
"largest_dup_signed_per_threshold": {
|
| 433 |
+
"0.5": 6.0,
|
| 434 |
+
"0.6": 2.0,
|
| 435 |
+
"0.7": 1.0,
|
| 436 |
+
"0.8": 1.0,
|
| 437 |
+
"0.9": 1.0
|
| 438 |
+
},
|
| 439 |
+
"largest_dup_signed_band": 1.25,
|
| 440 |
+
"largest_dup_signed_band_std": 0.0,
|
| 441 |
+
"dcs_signed_mean": 0.841914693514506,
|
| 442 |
+
"dcs_signed_std": 0.00036448183420947374,
|
| 443 |
+
"dedup_dcs_signed_per_threshold": {
|
| 444 |
+
"0.5": 0.7176240194538778,
|
| 445 |
+
"0.6": 0.8418438539293774,
|
| 446 |
+
"0.7": 0.841914693514506,
|
| 447 |
+
"0.8": 0.841914693514506,
|
| 448 |
+
"0.9": 0.841914693514506
|
| 449 |
+
},
|
| 450 |
+
"dedup_dcs_signed_band": 0.8418969836182238,
|
| 451 |
+
"dedup_dcs_signed_band_std": 0.00036504624021942595,
|
| 452 |
+
"matched_cos_signed_per_threshold": {
|
| 453 |
+
"0.5": 0.8373000621795654,
|
| 454 |
+
"0.6": 0.8419123689333597,
|
| 455 |
+
"0.7": 0.841914693514506,
|
| 456 |
+
"0.8": 0.841914693514506,
|
| 457 |
+
"0.9": 0.841914693514506
|
| 458 |
+
},
|
| 459 |
+
"matched_cos_signed_band": 0.8419141123692194,
|
| 460 |
+
"matched_cos_signed_band_std": 0.00036505370224472965,
|
| 461 |
+
"unique_frac_signed_per_threshold": {
|
| 462 |
+
"0.5": 0.8570692274305557,
|
| 463 |
+
"0.6": 0.9999186197916666,
|
| 464 |
+
"0.7": 1.0,
|
| 465 |
+
"0.8": 1.0,
|
| 466 |
+
"0.9": 1.0
|
| 467 |
+
},
|
| 468 |
+
"unique_frac_signed_band": 0.9999796549479166,
|
| 469 |
+
"unique_frac_signed_band_std": 0.0,
|
| 470 |
+
"matched_frac_signed_per_threshold": {
|
| 471 |
+
"0.5": 0.7945421006944445,
|
| 472 |
+
"0.6": 0.901828342013889,
|
| 473 |
+
"0.7": 0.8328721788194445,
|
| 474 |
+
"0.8": 0.7156304253472222,
|
| 475 |
+
"0.9": 0.5235731336805555
|
| 476 |
+
},
|
| 477 |
+
"matched_frac_signed_band": 0.7434760199652778,
|
| 478 |
+
"matched_frac_signed_band_std": 0.0010884761313991254,
|
| 479 |
+
"babel_mu1_abs_mean": 0.4411987364292145,
|
| 480 |
+
"babel_mu1_abs_std": 0.00014568990808618186,
|
| 481 |
+
"n_dup_abs_per_threshold": {
|
| 482 |
+
"0.5": 3315.0,
|
| 483 |
+
"0.6": 102.0,
|
| 484 |
+
"0.7": 21.333333333333332,
|
| 485 |
+
"0.8": 0.6666666666666666,
|
| 486 |
+
"0.9": 0.0
|
| 487 |
+
},
|
| 488 |
+
"n_dup_abs_band": 31.0,
|
| 489 |
+
"n_dup_abs_band_std": 3.905124837953327,
|
| 490 |
+
"n_dup_clusters_abs_per_threshold": {
|
| 491 |
+
"0.5": 1538.6666666666667,
|
| 492 |
+
"0.6": 51.0,
|
| 493 |
+
"0.7": 10.666666666666666,
|
| 494 |
+
"0.8": 0.3333333333333333,
|
| 495 |
+
"0.9": 0.0
|
| 496 |
+
},
|
| 497 |
+
"n_dup_clusters_abs_band": 15.5,
|
| 498 |
+
"n_dup_clusters_abs_band_std": 1.9525624189766635,
|
| 499 |
+
"largest_dup_abs_per_threshold": {
|
| 500 |
+
"0.5": 6.0,
|
| 501 |
+
"0.6": 2.0,
|
| 502 |
+
"0.7": 2.0,
|
| 503 |
+
"0.8": 1.3333333333333333,
|
| 504 |
+
"0.9": 1.0
|
| 505 |
+
},
|
| 506 |
+
"largest_dup_abs_band": 1.5833333333333333,
|
| 507 |
+
"largest_dup_abs_band_std": 0.14433756729740646,
|
| 508 |
+
"dcs_abs_mean": 0.8420622150103251,
|
| 509 |
+
"dcs_abs_std": 0.00038491065000425916,
|
| 510 |
+
"dedup_dcs_abs_per_threshold": {
|
| 511 |
+
"0.5": 0.7166096606524661,
|
| 512 |
+
"0.6": 0.8391576396501881,
|
| 513 |
+
"0.7": 0.8415304785626682,
|
| 514 |
+
"0.8": 0.8420676062451093,
|
| 515 |
+
"0.9": 0.8420622150103251
|
| 516 |
+
},
|
| 517 |
+
"dedup_dcs_abs_band": 0.8412044848670727,
|
| 518 |
+
"dedup_dcs_abs_band_std": 0.0003996889687014163,
|
| 519 |
+
"matched_cos_abs_per_threshold": {
|
| 520 |
+
"0.5": 0.8377072811126709,
|
| 521 |
+
"0.6": 0.8426549832026163,
|
| 522 |
+
"0.7": 0.8422616124153137,
|
| 523 |
+
"0.8": 0.8420904477437338,
|
| 524 |
+
"0.9": 0.8420622150103251
|
| 525 |
+
},
|
| 526 |
+
"matched_cos_abs_band": 0.8422673145929972,
|
| 527 |
+
"matched_cos_abs_band_std": 0.00038631035693890634,
|
| 528 |
+
"unique_frac_abs_per_threshold": {
|
| 529 |
+
"0.5": 0.8554416232638888,
|
| 530 |
+
"0.6": 0.995849609375,
|
| 531 |
+
"0.7": 0.9991319444444443,
|
| 532 |
+
"0.8": 0.999972873263889,
|
| 533 |
+
"0.9": 1.0
|
| 534 |
+
},
|
| 535 |
+
"unique_frac_abs_band": 0.9987386067708333,
|
| 536 |
+
"unique_frac_abs_band_std": 7.944996822004592e-05,
|
| 537 |
+
"matched_frac_abs_per_threshold": {
|
| 538 |
+
"0.5": 0.7930772569444445,
|
| 539 |
+
"0.6": 0.8975965711805555,
|
| 540 |
+
"0.7": 0.8321126302083334,
|
| 541 |
+
"0.8": 0.7160915798611112,
|
| 542 |
+
"0.9": 0.5237358940972223
|
| 543 |
+
},
|
| 544 |
+
"matched_frac_abs_band": 0.7423841688368056,
|
| 545 |
+
"matched_frac_abs_band_std": 0.0012332413623357275,
|
| 546 |
+
"alive_source": "stats",
|
| 547 |
+
"frac_denominator": "d_sae",
|
| 548 |
+
"d_sae": 12288
|
| 549 |
+
}
|
| 550 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 16,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 0,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 6.5,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 18,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_samples": 20000,
|
| 3 |
+
"n_positions": 20480000,
|
| 4 |
+
"l0_per_token_0": 68.38207143554688,
|
| 5 |
+
"l0_per_token_0.001": 68.3819552734375,
|
| 6 |
+
"l0_per_token_0.01": 68.38150571289063,
|
| 7 |
+
"l0_per_token_0.05": 68.37984819335938,
|
| 8 |
+
"l0_per_token_0.1": 68.37665571289062,
|
| 9 |
+
"l0_per_token_0.5": 68.29212431640624,
|
| 10 |
+
"l0_per_token_1.0": 63.50503798828125,
|
| 11 |
+
"l1_per_token": 255.81543037355635,
|
| 12 |
+
"r2_score": 0.9321300780911635,
|
| 13 |
+
"l2_loss": 1.5917148652048885
|
| 14 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_64/seed0/stats.fineweb.split_train.seed_no/meta.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "fineweb",
|
| 3 |
+
"split": "train",
|
| 4 |
+
"hf_dataset": "HuggingFaceFW/fineweb",
|
| 5 |
+
"hf_dataset_config": null,
|
| 6 |
+
"hf_dataset_text_field": "text",
|
| 7 |
+
"context_length": 1024,
|
| 8 |
+
"num_samples_requested": 20000,
|
| 9 |
+
"num_samples_actual": 20000,
|
| 10 |
+
"batch_seqs": 32,
|
| 11 |
+
"stats_top_k": 100,
|
| 12 |
+
"stream_seed": 0,
|
| 13 |
+
"sparsity_thresholds": [
|
| 14 |
+
0.001,
|
| 15 |
+
0.01,
|
| 16 |
+
0.05,
|
| 17 |
+
0.1,
|
| 18 |
+
0.5,
|
| 19 |
+
1.0
|
| 20 |
+
],
|
| 21 |
+
"model_hf_name": "openai-community/gpt2",
|
| 22 |
+
"hook_target_name": "transformer.h.8",
|
| 23 |
+
"tokenizer_hf_name": "openai-community/gpt2",
|
| 24 |
+
"d_sae": 12288,
|
| 25 |
+
"d_in": 768
|
| 26 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 16,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 1,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 6.5,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 18,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_samples": 20000,
|
| 3 |
+
"n_positions": 20480000,
|
| 4 |
+
"l0_per_token_0": 68.27429501953125,
|
| 5 |
+
"l0_per_token_0.001": 68.27426918945312,
|
| 6 |
+
"l0_per_token_0.01": 68.27396088867188,
|
| 7 |
+
"l0_per_token_0.05": 68.27259619140625,
|
| 8 |
+
"l0_per_token_0.1": 68.269659375,
|
| 9 |
+
"l0_per_token_0.5": 68.191526171875,
|
| 10 |
+
"l0_per_token_1.0": 63.4634873046875,
|
| 11 |
+
"l1_per_token": 255.2836143987286,
|
| 12 |
+
"r2_score": 0.9322215810946727,
|
| 13 |
+
"l2_loss": 1.5895688970528683
|
| 14 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_64/seed1/stats.fineweb.split_train.seed_no/meta.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "fineweb",
|
| 3 |
+
"split": "train",
|
| 4 |
+
"hf_dataset": "HuggingFaceFW/fineweb",
|
| 5 |
+
"hf_dataset_config": null,
|
| 6 |
+
"hf_dataset_text_field": "text",
|
| 7 |
+
"context_length": 1024,
|
| 8 |
+
"num_samples_requested": 20000,
|
| 9 |
+
"num_samples_actual": 20000,
|
| 10 |
+
"batch_seqs": 32,
|
| 11 |
+
"stats_top_k": 100,
|
| 12 |
+
"stream_seed": 0,
|
| 13 |
+
"sparsity_thresholds": [
|
| 14 |
+
0.001,
|
| 15 |
+
0.01,
|
| 16 |
+
0.05,
|
| 17 |
+
0.1,
|
| 18 |
+
0.5,
|
| 19 |
+
1.0
|
| 20 |
+
],
|
| 21 |
+
"model_hf_name": "openai-community/gpt2",
|
| 22 |
+
"hook_target_name": "transformer.h.8",
|
| 23 |
+
"tokenizer_hf_name": "openai-community/gpt2",
|
| 24 |
+
"d_sae": 12288,
|
| 25 |
+
"d_in": 768
|
| 26 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 16,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 2,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 6.5,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 18,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_samples": 20000,
|
| 3 |
+
"n_positions": 20480000,
|
| 4 |
+
"l0_per_token_0": 68.547587109375,
|
| 5 |
+
"l0_per_token_0.001": 68.5475615234375,
|
| 6 |
+
"l0_per_token_0.01": 68.54738310546875,
|
| 7 |
+
"l0_per_token_0.05": 68.54639868164062,
|
| 8 |
+
"l0_per_token_0.1": 68.54444653320313,
|
| 9 |
+
"l0_per_token_0.5": 68.47152568359375,
|
| 10 |
+
"l0_per_token_1.0": 63.59962797851563,
|
| 11 |
+
"l1_per_token": 256.91514224984536,
|
| 12 |
+
"r2_score": 0.9321723959031226,
|
| 13 |
+
"l2_loss": 1.5907224095122363
|
| 14 |
+
}
|
lm/gpt2-small_layer8/consdl/x16/l0_64/seed2/stats.fineweb.split_train.seed_no/meta.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "fineweb",
|
| 3 |
+
"split": "train",
|
| 4 |
+
"hf_dataset": "HuggingFaceFW/fineweb",
|
| 5 |
+
"hf_dataset_config": null,
|
| 6 |
+
"hf_dataset_text_field": "text",
|
| 7 |
+
"context_length": 1024,
|
| 8 |
+
"num_samples_requested": 20000,
|
| 9 |
+
"num_samples_actual": 20000,
|
| 10 |
+
"batch_seqs": 32,
|
| 11 |
+
"stats_top_k": 100,
|
| 12 |
+
"stream_seed": 0,
|
| 13 |
+
"sparsity_thresholds": [
|
| 14 |
+
0.001,
|
| 15 |
+
0.01,
|
| 16 |
+
0.05,
|
| 17 |
+
0.1,
|
| 18 |
+
0.5,
|
| 19 |
+
1.0
|
| 20 |
+
],
|
| 21 |
+
"model_hf_name": "openai-community/gpt2",
|
| 22 |
+
"hook_target_name": "transformer.h.8",
|
| 23 |
+
"tokenizer_hf_name": "openai-community/gpt2",
|
| 24 |
+
"d_sae": 12288,
|
| 25 |
+
"d_in": 768
|
| 26 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_128/code_metrics_step_final.json
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": 1,
|
| 3 |
+
"step": "final",
|
| 4 |
+
"n_tokens": 600000,
|
| 5 |
+
"thresholds": [
|
| 6 |
+
0.5,
|
| 7 |
+
0.6,
|
| 8 |
+
0.7,
|
| 9 |
+
0.8,
|
| 10 |
+
0.9
|
| 11 |
+
],
|
| 12 |
+
"seeds": [
|
| 13 |
+
"seed0",
|
| 14 |
+
"seed1",
|
| 15 |
+
"seed2"
|
| 16 |
+
],
|
| 17 |
+
"per_seed": {
|
| 18 |
+
"seed0": {
|
| 19 |
+
"l0": 127.844745,
|
| 20 |
+
"recon_r2": 0.9415145041736372,
|
| 21 |
+
"l1_per_token": 393.4831958380243
|
| 22 |
+
},
|
| 23 |
+
"seed1": {
|
| 24 |
+
"l0": 127.82662833333333,
|
| 25 |
+
"recon_r2": 0.9413935058689876,
|
| 26 |
+
"l1_per_token": 392.6340377780998
|
| 27 |
+
},
|
| 28 |
+
"seed2": {
|
| 29 |
+
"l0": 127.94816,
|
| 30 |
+
"recon_r2": 0.9412829085142881,
|
| 31 |
+
"l1_per_token": 392.29133534548754
|
| 32 |
+
}
|
| 33 |
+
},
|
| 34 |
+
"per_pair": {
|
| 35 |
+
"seed0\u2194seed1": {
|
| 36 |
+
"alive_source": "stats",
|
| 37 |
+
"mcc_signed": 0.8384991348521575,
|
| 38 |
+
"mcc_abs": 0.8385107009141827,
|
| 39 |
+
"act_iou": 0.5043471564033918,
|
| 40 |
+
"act_iou_n_pairs": 3072,
|
| 41 |
+
"input_r2": 0.9880527451632783,
|
| 42 |
+
"matched_mcc_signed": {
|
| 43 |
+
"0.6": 0.8957322305283675,
|
| 44 |
+
"0.7": 0.905955677007102,
|
| 45 |
+
"0.8": 0.9195577963035144,
|
| 46 |
+
"0.9": 0.9397493154646301
|
| 47 |
+
},
|
| 48 |
+
"matched_mcc_abs": {
|
| 49 |
+
"0.6": 0.8954973345640642,
|
| 50 |
+
"0.7": 0.904681714997588,
|
| 51 |
+
"0.8": 0.9186516445179674,
|
| 52 |
+
"0.9": 0.9396600462816262
|
| 53 |
+
}
|
| 54 |
+
},
|
| 55 |
+
"seed0\u2194seed2": {
|
| 56 |
+
"alive_source": "stats",
|
| 57 |
+
"mcc_signed": 0.8343279700795557,
|
| 58 |
+
"mcc_abs": 0.834332741334677,
|
| 59 |
+
"act_iou": 0.49883932105666623,
|
| 60 |
+
"act_iou_n_pairs": 3072,
|
| 61 |
+
"input_r2": 0.9878843753705879,
|
| 62 |
+
"matched_mcc_signed": {
|
| 63 |
+
"0.6": 0.8940291203862878,
|
| 64 |
+
"0.7": 0.9052258916690518,
|
| 65 |
+
"0.8": 0.9194332997748622,
|
| 66 |
+
"0.9": 0.9378714317629683
|
| 67 |
+
},
|
| 68 |
+
"matched_mcc_abs": {
|
| 69 |
+
"0.6": 0.892954798378118,
|
| 70 |
+
"0.7": 0.902677393241714,
|
| 71 |
+
"0.8": 0.9187113669722669,
|
| 72 |
+
"0.9": 0.937871431762968
|
| 73 |
+
}
|
| 74 |
+
},
|
| 75 |
+
"seed1\u2194seed2": {
|
| 76 |
+
"alive_source": "stats",
|
| 77 |
+
"mcc_signed": 0.8358634571464081,
|
| 78 |
+
"mcc_abs": 0.8358725738918551,
|
| 79 |
+
"act_iou": 0.4990281240433236,
|
| 80 |
+
"act_iou_n_pairs": 3072,
|
| 81 |
+
"input_r2": 0.9878902078566736,
|
| 82 |
+
"matched_mcc_signed": {
|
| 83 |
+
"0.6": 0.8962142435565161,
|
| 84 |
+
"0.7": 0.9058219536751124,
|
| 85 |
+
"0.8": 0.9194503320038773,
|
| 86 |
+
"0.9": 0.9381452531661661
|
| 87 |
+
},
|
| 88 |
+
"matched_mcc_abs": {
|
| 89 |
+
"0.6": 0.8950302618959097,
|
| 90 |
+
"0.7": 0.9037448659876602,
|
| 91 |
+
"0.8": 0.9182291337122168,
|
| 92 |
+
"0.9": 0.9375700971776684
|
| 93 |
+
}
|
| 94 |
+
}
|
| 95 |
+
},
|
| 96 |
+
"summary": {
|
| 97 |
+
"n_seeds": 3,
|
| 98 |
+
"n_pairs": 3,
|
| 99 |
+
"report_band": [
|
| 100 |
+
0.6,
|
| 101 |
+
0.7,
|
| 102 |
+
0.8,
|
| 103 |
+
0.9
|
| 104 |
+
],
|
| 105 |
+
"l0_mean": 127.8731777777778,
|
| 106 |
+
"l0_std": 0.06556526244754074,
|
| 107 |
+
"recon_r2_mean": 0.9413969728523043,
|
| 108 |
+
"recon_r2_std": 0.00011583674864796442,
|
| 109 |
+
"l1_per_token_mean": 392.8028563205373,
|
| 110 |
+
"l1_per_token_std": 0.6136021786308381,
|
| 111 |
+
"input_r2_mean": 0.9879424427968466,
|
| 112 |
+
"input_r2_std": 9.556915560898149e-05,
|
| 113 |
+
"mcc_signed_mean": 0.8362301873593738,
|
| 114 |
+
"mcc_signed_std": 0.0021096260751315627,
|
| 115 |
+
"mcc_abs_mean": 0.8362386720469049,
|
| 116 |
+
"mcc_abs_std": 0.002112902614022678,
|
| 117 |
+
"matched_mcc_signed_per_threshold": {
|
| 118 |
+
"0.6": 0.8953251981570571,
|
| 119 |
+
"0.7": 0.9056678407837554,
|
| 120 |
+
"0.8": 0.919480476027418,
|
| 121 |
+
"0.9": 0.9385886667979215
|
| 122 |
+
},
|
| 123 |
+
"matched_mcc_signed_band": 0.914765545441538,
|
| 124 |
+
"matched_mcc_abs_per_threshold": {
|
| 125 |
+
"0.6": 0.8944941316126972,
|
| 126 |
+
"0.7": 0.9037013247423208,
|
| 127 |
+
"0.8": 0.9185307150674836,
|
| 128 |
+
"0.9": 0.9383671917407543
|
| 129 |
+
},
|
| 130 |
+
"matched_mcc_abs_band": 0.913773340790814,
|
| 131 |
+
"act_iou_mean": 0.5007382005011273,
|
| 132 |
+
"act_iou_std": 0.0031268728260532867,
|
| 133 |
+
"act_iou_n_pairs": 3072.0,
|
| 134 |
+
"alive_source": "stats"
|
| 135 |
+
}
|
| 136 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_128/dict_metrics_step_final.json
ADDED
|
@@ -0,0 +1,550 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": 2,
|
| 3 |
+
"step": "final",
|
| 4 |
+
"linkage": "complete",
|
| 5 |
+
"thresholds": [
|
| 6 |
+
0.5,
|
| 7 |
+
0.6,
|
| 8 |
+
0.7,
|
| 9 |
+
0.8,
|
| 10 |
+
0.9
|
| 11 |
+
],
|
| 12 |
+
"seeds": [
|
| 13 |
+
"seed0",
|
| 14 |
+
"seed1",
|
| 15 |
+
"seed2"
|
| 16 |
+
],
|
| 17 |
+
"per_seed": {
|
| 18 |
+
"seed0": {
|
| 19 |
+
"d_sae": 3072,
|
| 20 |
+
"d_in": 768,
|
| 21 |
+
"l0": 126,
|
| 22 |
+
"l0_raw": 125.704651171875,
|
| 23 |
+
"recon_r2": 0.9411897999822579,
|
| 24 |
+
"l1_per_token": 391.79768565147,
|
| 25 |
+
"stats_source": "stats",
|
| 26 |
+
"n_dead": 0,
|
| 27 |
+
"dead_rate": 0.0,
|
| 28 |
+
"alive_source": "stats",
|
| 29 |
+
"n_dense": 51,
|
| 30 |
+
"dense_rate": 0.0166015625,
|
| 31 |
+
"dense_threshold": 0.1,
|
| 32 |
+
"n_dup_signed": {
|
| 33 |
+
"0.5": 229,
|
| 34 |
+
"0.6": 0,
|
| 35 |
+
"0.7": 0,
|
| 36 |
+
"0.8": 0,
|
| 37 |
+
"0.9": 0
|
| 38 |
+
},
|
| 39 |
+
"n_dup_clusters_signed": {
|
| 40 |
+
"0.5": 109,
|
| 41 |
+
"0.6": 0,
|
| 42 |
+
"0.7": 0,
|
| 43 |
+
"0.8": 0,
|
| 44 |
+
"0.9": 0
|
| 45 |
+
},
|
| 46 |
+
"largest_dup_signed": {
|
| 47 |
+
"0.5": 3,
|
| 48 |
+
"0.6": 1,
|
| 49 |
+
"0.7": 1,
|
| 50 |
+
"0.8": 1,
|
| 51 |
+
"0.9": 1
|
| 52 |
+
},
|
| 53 |
+
"babel_mu1_signed": 0.33925139904022217,
|
| 54 |
+
"n_dup_abs": {
|
| 55 |
+
"0.5": 305,
|
| 56 |
+
"0.6": 88,
|
| 57 |
+
"0.7": 54,
|
| 58 |
+
"0.8": 10,
|
| 59 |
+
"0.9": 0
|
| 60 |
+
},
|
| 61 |
+
"n_dup_clusters_abs": {
|
| 62 |
+
"0.5": 147,
|
| 63 |
+
"0.6": 44,
|
| 64 |
+
"0.7": 27,
|
| 65 |
+
"0.8": 5,
|
| 66 |
+
"0.9": 0
|
| 67 |
+
},
|
| 68 |
+
"largest_dup_abs": {
|
| 69 |
+
"0.5": 3,
|
| 70 |
+
"0.6": 2,
|
| 71 |
+
"0.7": 2,
|
| 72 |
+
"0.8": 2,
|
| 73 |
+
"0.9": 1
|
| 74 |
+
},
|
| 75 |
+
"babel_mu1_abs": 0.35641753673553467
|
| 76 |
+
},
|
| 77 |
+
"seed1": {
|
| 78 |
+
"d_sae": 3072,
|
| 79 |
+
"d_in": 768,
|
| 80 |
+
"l0": 126,
|
| 81 |
+
"l0_raw": 125.63611196289062,
|
| 82 |
+
"recon_r2": 0.9410739181284478,
|
| 83 |
+
"l1_per_token": 390.88694813323394,
|
| 84 |
+
"stats_source": "stats",
|
| 85 |
+
"n_dead": 0,
|
| 86 |
+
"dead_rate": 0.0,
|
| 87 |
+
"alive_source": "stats",
|
| 88 |
+
"n_dense": 53,
|
| 89 |
+
"dense_rate": 0.017252604166666668,
|
| 90 |
+
"dense_threshold": 0.1,
|
| 91 |
+
"n_dup_signed": {
|
| 92 |
+
"0.5": 222,
|
| 93 |
+
"0.6": 0,
|
| 94 |
+
"0.7": 0,
|
| 95 |
+
"0.8": 0,
|
| 96 |
+
"0.9": 0
|
| 97 |
+
},
|
| 98 |
+
"n_dup_clusters_signed": {
|
| 99 |
+
"0.5": 105,
|
| 100 |
+
"0.6": 0,
|
| 101 |
+
"0.7": 0,
|
| 102 |
+
"0.8": 0,
|
| 103 |
+
"0.9": 0
|
| 104 |
+
},
|
| 105 |
+
"largest_dup_signed": {
|
| 106 |
+
"0.5": 4,
|
| 107 |
+
"0.6": 1,
|
| 108 |
+
"0.7": 1,
|
| 109 |
+
"0.8": 1,
|
| 110 |
+
"0.9": 1
|
| 111 |
+
},
|
| 112 |
+
"babel_mu1_signed": 0.3414287567138672,
|
| 113 |
+
"n_dup_abs": {
|
| 114 |
+
"0.5": 296,
|
| 115 |
+
"0.6": 80,
|
| 116 |
+
"0.7": 46,
|
| 117 |
+
"0.8": 10,
|
| 118 |
+
"0.9": 0
|
| 119 |
+
},
|
| 120 |
+
"n_dup_clusters_abs": {
|
| 121 |
+
"0.5": 143,
|
| 122 |
+
"0.6": 40,
|
| 123 |
+
"0.7": 23,
|
| 124 |
+
"0.8": 5,
|
| 125 |
+
"0.9": 0
|
| 126 |
+
},
|
| 127 |
+
"largest_dup_abs": {
|
| 128 |
+
"0.5": 4,
|
| 129 |
+
"0.6": 2,
|
| 130 |
+
"0.7": 2,
|
| 131 |
+
"0.8": 2,
|
| 132 |
+
"0.9": 1
|
| 133 |
+
},
|
| 134 |
+
"babel_mu1_abs": 0.35803985595703125
|
| 135 |
+
},
|
| 136 |
+
"seed2": {
|
| 137 |
+
"d_sae": 3072,
|
| 138 |
+
"d_in": 768,
|
| 139 |
+
"l0": 126,
|
| 140 |
+
"l0_raw": 125.77726040039063,
|
| 141 |
+
"recon_r2": 0.9409605591020928,
|
| 142 |
+
"l1_per_token": 390.5783639531069,
|
| 143 |
+
"stats_source": "stats",
|
| 144 |
+
"n_dead": 0,
|
| 145 |
+
"dead_rate": 0.0,
|
| 146 |
+
"alive_source": "stats",
|
| 147 |
+
"n_dense": 55,
|
| 148 |
+
"dense_rate": 0.017903645833333332,
|
| 149 |
+
"dense_threshold": 0.1,
|
| 150 |
+
"n_dup_signed": {
|
| 151 |
+
"0.5": 212,
|
| 152 |
+
"0.6": 0,
|
| 153 |
+
"0.7": 0,
|
| 154 |
+
"0.8": 0,
|
| 155 |
+
"0.9": 0
|
| 156 |
+
},
|
| 157 |
+
"n_dup_clusters_signed": {
|
| 158 |
+
"0.5": 101,
|
| 159 |
+
"0.6": 0,
|
| 160 |
+
"0.7": 0,
|
| 161 |
+
"0.8": 0,
|
| 162 |
+
"0.9": 0
|
| 163 |
+
},
|
| 164 |
+
"largest_dup_signed": {
|
| 165 |
+
"0.5": 4,
|
| 166 |
+
"0.6": 1,
|
| 167 |
+
"0.7": 1,
|
| 168 |
+
"0.8": 1,
|
| 169 |
+
"0.9": 1
|
| 170 |
+
},
|
| 171 |
+
"babel_mu1_signed": 0.3400411605834961,
|
| 172 |
+
"n_dup_abs": {
|
| 173 |
+
"0.5": 272,
|
| 174 |
+
"0.6": 78,
|
| 175 |
+
"0.7": 44,
|
| 176 |
+
"0.8": 6,
|
| 177 |
+
"0.9": 0
|
| 178 |
+
},
|
| 179 |
+
"n_dup_clusters_abs": {
|
| 180 |
+
"0.5": 131,
|
| 181 |
+
"0.6": 39,
|
| 182 |
+
"0.7": 22,
|
| 183 |
+
"0.8": 3,
|
| 184 |
+
"0.9": 0
|
| 185 |
+
},
|
| 186 |
+
"largest_dup_abs": {
|
| 187 |
+
"0.5": 4,
|
| 188 |
+
"0.6": 2,
|
| 189 |
+
"0.7": 2,
|
| 190 |
+
"0.8": 2,
|
| 191 |
+
"0.9": 1
|
| 192 |
+
},
|
| 193 |
+
"babel_mu1_abs": 0.3550417423248291
|
| 194 |
+
}
|
| 195 |
+
},
|
| 196 |
+
"per_pair": {
|
| 197 |
+
"seed0\u2194seed1": {
|
| 198 |
+
"dcs_signed": 0.8878882527351379,
|
| 199 |
+
"dcs_abs": 0.8885080218315125,
|
| 200 |
+
"frac_sign_flipped": 0.012044270833333334,
|
| 201 |
+
"alive_source": "stats",
|
| 202 |
+
"frac_denominator": "d_sae",
|
| 203 |
+
"n_alive_a": 3072,
|
| 204 |
+
"n_alive_b": 3072,
|
| 205 |
+
"matched_cos_signed": {
|
| 206 |
+
"0.5": 0.8854705691337585,
|
| 207 |
+
"0.6": 0.8878882527351379,
|
| 208 |
+
"0.7": 0.8878882527351379,
|
| 209 |
+
"0.8": 0.8878882527351379,
|
| 210 |
+
"0.9": 0.8878882527351379
|
| 211 |
+
},
|
| 212 |
+
"unique_frac_signed": {
|
| 213 |
+
"0.5": 0.96142578125,
|
| 214 |
+
"0.6": 1.0,
|
| 215 |
+
"0.7": 1.0,
|
| 216 |
+
"0.8": 1.0,
|
| 217 |
+
"0.9": 1.0
|
| 218 |
+
},
|
| 219 |
+
"dedup_dcs_signed": {
|
| 220 |
+
"0.5": 0.851314233703306,
|
| 221 |
+
"0.6": 0.8878882527351379,
|
| 222 |
+
"0.7": 0.8878882527351379,
|
| 223 |
+
"0.8": 0.8878882527351379,
|
| 224 |
+
"0.9": 0.8878882527351379
|
| 225 |
+
},
|
| 226 |
+
"matched_frac_signed": {
|
| 227 |
+
"0.5": 0.890625,
|
| 228 |
+
"0.6": 0.9153645833333334,
|
| 229 |
+
"0.7": 0.8896484375,
|
| 230 |
+
"0.8": 0.8453776041666666,
|
| 231 |
+
"0.9": 0.7454427083333334
|
| 232 |
+
},
|
| 233 |
+
"matched_cos_abs": {
|
| 234 |
+
"0.5": 0.8882692456245422,
|
| 235 |
+
"0.6": 0.8909647464752197,
|
| 236 |
+
"0.7": 0.8901378512382507,
|
| 237 |
+
"0.8": 0.8887239098548889,
|
| 238 |
+
"0.9": 0.8885080218315125
|
| 239 |
+
},
|
| 240 |
+
"unique_frac_abs": {
|
| 241 |
+
"0.5": 0.9493815104166666,
|
| 242 |
+
"0.6": 0.986328125,
|
| 243 |
+
"0.7": 0.9918619791666666,
|
| 244 |
+
"0.8": 0.9983723958333334,
|
| 245 |
+
"0.9": 1.0
|
| 246 |
+
},
|
| 247 |
+
"dedup_dcs_abs": {
|
| 248 |
+
"0.5": 0.8433063980677009,
|
| 249 |
+
"0.6": 0.8787835878320038,
|
| 250 |
+
"0.7": 0.8828938908603352,
|
| 251 |
+
"0.8": 0.8872774191161928,
|
| 252 |
+
"0.9": 0.8885080218315125
|
| 253 |
+
},
|
| 254 |
+
"matched_frac_abs": {
|
| 255 |
+
"0.5": 0.8802083333333334,
|
| 256 |
+
"0.6": 0.904296875,
|
| 257 |
+
"0.7": 0.8841145833333334,
|
| 258 |
+
"0.8": 0.845703125,
|
| 259 |
+
"0.9": 0.74609375
|
| 260 |
+
}
|
| 261 |
+
},
|
| 262 |
+
"seed0\u2194seed2": {
|
| 263 |
+
"dcs_signed": 0.8845382332801819,
|
| 264 |
+
"dcs_abs": 0.8851227760314941,
|
| 265 |
+
"frac_sign_flipped": 0.014322916666666666,
|
| 266 |
+
"alive_source": "stats",
|
| 267 |
+
"frac_denominator": "d_sae",
|
| 268 |
+
"n_alive_a": 3072,
|
| 269 |
+
"n_alive_b": 3072,
|
| 270 |
+
"matched_cos_signed": {
|
| 271 |
+
"0.5": 0.8836352229118347,
|
| 272 |
+
"0.6": 0.8845381736755371,
|
| 273 |
+
"0.7": 0.8845381736755371,
|
| 274 |
+
"0.8": 0.8845381736755371,
|
| 275 |
+
"0.9": 0.8845381736755371
|
| 276 |
+
},
|
| 277 |
+
"unique_frac_signed": {
|
| 278 |
+
"0.5": 0.96240234375,
|
| 279 |
+
"0.6": 1.0,
|
| 280 |
+
"0.7": 1.0,
|
| 281 |
+
"0.8": 1.0,
|
| 282 |
+
"0.9": 1.0
|
| 283 |
+
},
|
| 284 |
+
"dedup_dcs_signed": {
|
| 285 |
+
"0.5": 0.8504126095504034,
|
| 286 |
+
"0.6": 0.8845381736755371,
|
| 287 |
+
"0.7": 0.8845381736755371,
|
| 288 |
+
"0.8": 0.8845381736755371,
|
| 289 |
+
"0.9": 0.8845381736755371
|
| 290 |
+
},
|
| 291 |
+
"matched_frac_signed": {
|
| 292 |
+
"0.5": 0.8922526041666666,
|
| 293 |
+
"0.6": 0.908203125,
|
| 294 |
+
"0.7": 0.8824869791666666,
|
| 295 |
+
"0.8": 0.8382161458333334,
|
| 296 |
+
"0.9": 0.74609375
|
| 297 |
+
},
|
| 298 |
+
"matched_cos_abs": {
|
| 299 |
+
"0.5": 0.8872216939926147,
|
| 300 |
+
"0.6": 0.8873640298843384,
|
| 301 |
+
"0.7": 0.8865522742271423,
|
| 302 |
+
"0.8": 0.8858667016029358,
|
| 303 |
+
"0.9": 0.8851227760314941
|
| 304 |
+
},
|
| 305 |
+
"unique_frac_abs": {
|
| 306 |
+
"0.5": 0.9513346354166666,
|
| 307 |
+
"0.6": 0.9864908854166666,
|
| 308 |
+
"0.7": 0.9920247395833334,
|
| 309 |
+
"0.8": 0.9986979166666666,
|
| 310 |
+
"0.9": 1.0
|
| 311 |
+
},
|
| 312 |
+
"dedup_dcs_abs": {
|
| 313 |
+
"0.5": 0.8440447267882215,
|
| 314 |
+
"0.6": 0.8753765275275024,
|
| 315 |
+
"0.7": 0.8794817889671929,
|
| 316 |
+
"0.8": 0.8847132293352236,
|
| 317 |
+
"0.9": 0.8851227760314941
|
| 318 |
+
},
|
| 319 |
+
"matched_frac_abs": {
|
| 320 |
+
"0.5": 0.8831380208333334,
|
| 321 |
+
"0.6": 0.8955078125,
|
| 322 |
+
"0.7": 0.8779296875,
|
| 323 |
+
"0.8": 0.8382161458333334,
|
| 324 |
+
"0.9": 0.74609375
|
| 325 |
+
}
|
| 326 |
+
},
|
| 327 |
+
"seed1\u2194seed2": {
|
| 328 |
+
"dcs_signed": 0.8841626644134521,
|
| 329 |
+
"dcs_abs": 0.884857714176178,
|
| 330 |
+
"frac_sign_flipped": 0.012369791666666666,
|
| 331 |
+
"alive_source": "stats",
|
| 332 |
+
"frac_denominator": "d_sae",
|
| 333 |
+
"n_alive_a": 3072,
|
| 334 |
+
"n_alive_b": 3072,
|
| 335 |
+
"matched_cos_signed": {
|
| 336 |
+
"0.5": 0.8812102675437927,
|
| 337 |
+
"0.6": 0.8841626644134521,
|
| 338 |
+
"0.7": 0.8841626644134521,
|
| 339 |
+
"0.8": 0.8841626644134521,
|
| 340 |
+
"0.9": 0.8841626644134521
|
| 341 |
+
},
|
| 342 |
+
"unique_frac_signed": {
|
| 343 |
+
"0.5": 0.962890625,
|
| 344 |
+
"0.6": 1.0,
|
| 345 |
+
"0.7": 1.0,
|
| 346 |
+
"0.8": 1.0,
|
| 347 |
+
"0.9": 1.0
|
| 348 |
+
},
|
| 349 |
+
"dedup_dcs_signed": {
|
| 350 |
+
"0.5": 0.8485091052716598,
|
| 351 |
+
"0.6": 0.8841626644134521,
|
| 352 |
+
"0.7": 0.8841626644134521,
|
| 353 |
+
"0.8": 0.8841626644134521,
|
| 354 |
+
"0.9": 0.8841626644134521
|
| 355 |
+
},
|
| 356 |
+
"matched_frac_signed": {
|
| 357 |
+
"0.5": 0.8870442708333334,
|
| 358 |
+
"0.6": 0.9065755208333334,
|
| 359 |
+
"0.7": 0.8834635416666666,
|
| 360 |
+
"0.8": 0.8408203125,
|
| 361 |
+
"0.9": 0.7425130208333334
|
| 362 |
+
},
|
| 363 |
+
"matched_cos_abs": {
|
| 364 |
+
"0.5": 0.8841249346733093,
|
| 365 |
+
"0.6": 0.8862740993499756,
|
| 366 |
+
"0.7": 0.8853506445884705,
|
| 367 |
+
"0.8": 0.8853362202644348,
|
| 368 |
+
"0.9": 0.884857714176178
|
| 369 |
+
},
|
| 370 |
+
"unique_frac_abs": {
|
| 371 |
+
"0.5": 0.9521484375,
|
| 372 |
+
"0.6": 0.9871419270833334,
|
| 373 |
+
"0.7": 0.99267578125,
|
| 374 |
+
"0.8": 0.9986979166666666,
|
| 375 |
+
"0.9": 1.0
|
| 376 |
+
},
|
| 377 |
+
"dedup_dcs_abs": {
|
| 378 |
+
"0.5": 0.841818175103981,
|
| 379 |
+
"0.6": 0.8748783223563805,
|
| 380 |
+
"0.7": 0.878866142797051,
|
| 381 |
+
"0.8": 0.8841834387276322,
|
| 382 |
+
"0.9": 0.884857714176178
|
| 383 |
+
},
|
| 384 |
+
"matched_frac_abs": {
|
| 385 |
+
"0.5": 0.8772786458333334,
|
| 386 |
+
"0.6": 0.896484375,
|
| 387 |
+
"0.7": 0.8792317708333334,
|
| 388 |
+
"0.8": 0.841796875,
|
| 389 |
+
"0.9": 0.7425130208333334
|
| 390 |
+
}
|
| 391 |
+
}
|
| 392 |
+
},
|
| 393 |
+
"summary": {
|
| 394 |
+
"n_seeds": 3,
|
| 395 |
+
"n_pairs": 3,
|
| 396 |
+
"report_band": [
|
| 397 |
+
0.6,
|
| 398 |
+
0.7,
|
| 399 |
+
0.8,
|
| 400 |
+
0.9
|
| 401 |
+
],
|
| 402 |
+
"n_dead_mean": 0,
|
| 403 |
+
"n_dead_std": 0.0,
|
| 404 |
+
"n_dense_mean": 53,
|
| 405 |
+
"n_dense_std": 2.0,
|
| 406 |
+
"l0_mean": 126,
|
| 407 |
+
"l0_std": 0.0705839980006476,
|
| 408 |
+
"recon_r2_mean": 0.9410747590709327,
|
| 409 |
+
"recon_r2_std": 0.00011462275373120408,
|
| 410 |
+
"l1_per_token_mean": 391.0876659126036,
|
| 411 |
+
"l1_per_token_std": 0.6339574680063991,
|
| 412 |
+
"babel_mu1_signed_mean": 0.34024043877919513,
|
| 413 |
+
"babel_mu1_signed_std": 0.0011022728605957857,
|
| 414 |
+
"n_dup_signed_per_threshold": {
|
| 415 |
+
"0.5": 221.0,
|
| 416 |
+
"0.6": 0.0,
|
| 417 |
+
"0.7": 0.0,
|
| 418 |
+
"0.8": 0.0,
|
| 419 |
+
"0.9": 0.0
|
| 420 |
+
},
|
| 421 |
+
"n_dup_signed_band": 0.0,
|
| 422 |
+
"n_dup_signed_band_std": 0.0,
|
| 423 |
+
"n_dup_clusters_signed_per_threshold": {
|
| 424 |
+
"0.5": 105.0,
|
| 425 |
+
"0.6": 0.0,
|
| 426 |
+
"0.7": 0.0,
|
| 427 |
+
"0.8": 0.0,
|
| 428 |
+
"0.9": 0.0
|
| 429 |
+
},
|
| 430 |
+
"n_dup_clusters_signed_band": 0.0,
|
| 431 |
+
"n_dup_clusters_signed_band_std": 0.0,
|
| 432 |
+
"largest_dup_signed_per_threshold": {
|
| 433 |
+
"0.5": 3.6666666666666665,
|
| 434 |
+
"0.6": 1.0,
|
| 435 |
+
"0.7": 1.0,
|
| 436 |
+
"0.8": 1.0,
|
| 437 |
+
"0.9": 1.0
|
| 438 |
+
},
|
| 439 |
+
"largest_dup_signed_band": 1.0,
|
| 440 |
+
"largest_dup_signed_band_std": 0.0,
|
| 441 |
+
"dcs_signed_mean": 0.8855297168095907,
|
| 442 |
+
"dcs_signed_std": 0.002051165955618181,
|
| 443 |
+
"dedup_dcs_signed_per_threshold": {
|
| 444 |
+
"0.5": 0.8500786495084564,
|
| 445 |
+
"0.6": 0.8855296969413757,
|
| 446 |
+
"0.7": 0.8855296969413757,
|
| 447 |
+
"0.8": 0.8855296969413757,
|
| 448 |
+
"0.9": 0.8855296969413757
|
| 449 |
+
},
|
| 450 |
+
"dedup_dcs_signed_band": 0.8855296969413757,
|
| 451 |
+
"dedup_dcs_signed_band_std": 0.002051180361571077,
|
| 452 |
+
"matched_cos_signed_per_threshold": {
|
| 453 |
+
"0.5": 0.8834386865297953,
|
| 454 |
+
"0.6": 0.8855296969413757,
|
| 455 |
+
"0.7": 0.8855296969413757,
|
| 456 |
+
"0.8": 0.8855296969413757,
|
| 457 |
+
"0.9": 0.8855296969413757
|
| 458 |
+
},
|
| 459 |
+
"matched_cos_signed_band": 0.8855296969413757,
|
| 460 |
+
"matched_cos_signed_band_std": 0.002051180361571077,
|
| 461 |
+
"unique_frac_signed_per_threshold": {
|
| 462 |
+
"0.5": 0.9622395833333334,
|
| 463 |
+
"0.6": 1.0,
|
| 464 |
+
"0.7": 1.0,
|
| 465 |
+
"0.8": 1.0,
|
| 466 |
+
"0.9": 1.0
|
| 467 |
+
},
|
| 468 |
+
"unique_frac_signed_band": 1.0,
|
| 469 |
+
"unique_frac_signed_band_std": 0.0,
|
| 470 |
+
"matched_frac_signed_per_threshold": {
|
| 471 |
+
"0.5": 0.8899739583333334,
|
| 472 |
+
"0.6": 0.9100477430555557,
|
| 473 |
+
"0.7": 0.8851996527777777,
|
| 474 |
+
"0.8": 0.8414713541666666,
|
| 475 |
+
"0.9": 0.7446831597222223
|
| 476 |
+
},
|
| 477 |
+
"matched_frac_signed_band": 0.8453504774305556,
|
| 478 |
+
"matched_frac_signed_band_std": 0.003131111667736198,
|
| 479 |
+
"babel_mu1_abs_mean": 0.356499711672465,
|
| 480 |
+
"babel_mu1_abs_std": 0.0015007451076361096,
|
| 481 |
+
"n_dup_abs_per_threshold": {
|
| 482 |
+
"0.5": 291.0,
|
| 483 |
+
"0.6": 82.0,
|
| 484 |
+
"0.7": 48.0,
|
| 485 |
+
"0.8": 8.666666666666666,
|
| 486 |
+
"0.9": 0.0
|
| 487 |
+
},
|
| 488 |
+
"n_dup_abs_band": 34.666666666666664,
|
| 489 |
+
"n_dup_abs_band_std": 3.055050463303893,
|
| 490 |
+
"n_dup_clusters_abs_per_threshold": {
|
| 491 |
+
"0.5": 140.33333333333334,
|
| 492 |
+
"0.6": 41.0,
|
| 493 |
+
"0.7": 24.0,
|
| 494 |
+
"0.8": 4.333333333333333,
|
| 495 |
+
"0.9": 0.0
|
| 496 |
+
},
|
| 497 |
+
"n_dup_clusters_abs_band": 17.333333333333332,
|
| 498 |
+
"n_dup_clusters_abs_band_std": 1.5275252316519465,
|
| 499 |
+
"largest_dup_abs_per_threshold": {
|
| 500 |
+
"0.5": 3.6666666666666665,
|
| 501 |
+
"0.6": 2.0,
|
| 502 |
+
"0.7": 2.0,
|
| 503 |
+
"0.8": 2.0,
|
| 504 |
+
"0.9": 1.0
|
| 505 |
+
},
|
| 506 |
+
"largest_dup_abs_band": 1.75,
|
| 507 |
+
"largest_dup_abs_band_std": 0.0,
|
| 508 |
+
"dcs_abs_mean": 0.8861628373463949,
|
| 509 |
+
"dcs_abs_std": 0.0020353088583094994,
|
| 510 |
+
"dedup_dcs_abs_per_threshold": {
|
| 511 |
+
"0.5": 0.8430564333199678,
|
| 512 |
+
"0.6": 0.8763461459052956,
|
| 513 |
+
"0.7": 0.8804139408748597,
|
| 514 |
+
"0.8": 0.8853913623930162,
|
| 515 |
+
"0.9": 0.8861628373463949
|
| 516 |
+
},
|
| 517 |
+
"dedup_dcs_abs_band": 0.8820785716298917,
|
| 518 |
+
"dedup_dcs_abs_band_std": 0.001995054879145602,
|
| 519 |
+
"matched_cos_abs_per_threshold": {
|
| 520 |
+
"0.5": 0.8865386247634888,
|
| 521 |
+
"0.6": 0.8882009585698446,
|
| 522 |
+
"0.7": 0.8873469233512878,
|
| 523 |
+
"0.8": 0.8866422772407532,
|
| 524 |
+
"0.9": 0.8861628373463949
|
| 525 |
+
},
|
| 526 |
+
"matched_cos_abs_band": 0.8870882491270702,
|
| 527 |
+
"matched_cos_abs_band_std": 0.0021952477216257327,
|
| 528 |
+
"unique_frac_abs_per_threshold": {
|
| 529 |
+
"0.5": 0.950954861111111,
|
| 530 |
+
"0.6": 0.9866536458333334,
|
| 531 |
+
"0.7": 0.9921875,
|
| 532 |
+
"0.8": 0.9985894097222222,
|
| 533 |
+
"0.9": 1.0
|
| 534 |
+
},
|
| 535 |
+
"unique_frac_abs_band": 0.994357638888889,
|
| 536 |
+
"unique_frac_abs_band_std": 0.0002486206431725214,
|
| 537 |
+
"matched_frac_abs_per_threshold": {
|
| 538 |
+
"0.5": 0.8802083333333334,
|
| 539 |
+
"0.6": 0.8987630208333334,
|
| 540 |
+
"0.7": 0.8804253472222223,
|
| 541 |
+
"0.8": 0.8419053819444445,
|
| 542 |
+
"0.9": 0.7449001736111112
|
| 543 |
+
},
|
| 544 |
+
"matched_frac_abs_band": 0.8414984809027779,
|
| 545 |
+
"matched_frac_abs_band_std": 0.003090662747715338,
|
| 546 |
+
"alive_source": "stats",
|
| 547 |
+
"frac_denominator": "d_sae",
|
| 548 |
+
"d_sae": 3072
|
| 549 |
+
}
|
| 550 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 4,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 0,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 2.95,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 5,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_samples": 20000,
|
| 3 |
+
"n_positions": 20480000,
|
| 4 |
+
"l0_per_token_0": 125.704651171875,
|
| 5 |
+
"l0_per_token_0.001": 125.7046107421875,
|
| 6 |
+
"l0_per_token_0.01": 125.7044861328125,
|
| 7 |
+
"l0_per_token_0.05": 125.70372827148438,
|
| 8 |
+
"l0_per_token_0.1": 125.70257119140625,
|
| 9 |
+
"l0_per_token_0.5": 125.60522626953124,
|
| 10 |
+
"l0_per_token_1.0": 94.27361411132813,
|
| 11 |
+
"l1_per_token": 391.79768565147,
|
| 12 |
+
"r2_score": 0.9411897999822579,
|
| 13 |
+
"l2_loss": 1.3792423353551146
|
| 14 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_128/seed0/stats.fineweb.split_train.seed_no/meta.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "fineweb",
|
| 3 |
+
"split": "train",
|
| 4 |
+
"hf_dataset": "HuggingFaceFW/fineweb",
|
| 5 |
+
"hf_dataset_config": null,
|
| 6 |
+
"hf_dataset_text_field": "text",
|
| 7 |
+
"context_length": 1024,
|
| 8 |
+
"num_samples_requested": 20000,
|
| 9 |
+
"num_samples_actual": 20000,
|
| 10 |
+
"batch_seqs": 32,
|
| 11 |
+
"stats_top_k": 100,
|
| 12 |
+
"stream_seed": 0,
|
| 13 |
+
"sparsity_thresholds": [
|
| 14 |
+
0.001,
|
| 15 |
+
0.01,
|
| 16 |
+
0.05,
|
| 17 |
+
0.1,
|
| 18 |
+
0.5,
|
| 19 |
+
1.0
|
| 20 |
+
],
|
| 21 |
+
"model_hf_name": "openai-community/gpt2",
|
| 22 |
+
"hook_target_name": "transformer.h.8",
|
| 23 |
+
"tokenizer_hf_name": "openai-community/gpt2",
|
| 24 |
+
"d_sae": 3072,
|
| 25 |
+
"d_in": 768
|
| 26 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 4,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 1,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 2.95,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 5,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/l0_per_token.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_samples": 20000,
|
| 3 |
+
"n_positions": 20480000,
|
| 4 |
+
"l0_per_token_0": 125.63611196289062,
|
| 5 |
+
"l0_per_token_0.001": 125.63610112304687,
|
| 6 |
+
"l0_per_token_0.01": 125.63594052734375,
|
| 7 |
+
"l0_per_token_0.05": 125.635241796875,
|
| 8 |
+
"l0_per_token_0.1": 125.63404838867187,
|
| 9 |
+
"l0_per_token_0.5": 125.52499116210937,
|
| 10 |
+
"l0_per_token_1.0": 93.74265239257812,
|
| 11 |
+
"l1_per_token": 390.88694813323394,
|
| 12 |
+
"r2_score": 0.9410739181284478,
|
| 13 |
+
"l2_loss": 1.3819600468851936
|
| 14 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_128/seed1/stats.fineweb.split_train.seed_no/meta.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "fineweb",
|
| 3 |
+
"split": "train",
|
| 4 |
+
"hf_dataset": "HuggingFaceFW/fineweb",
|
| 5 |
+
"hf_dataset_config": null,
|
| 6 |
+
"hf_dataset_text_field": "text",
|
| 7 |
+
"context_length": 1024,
|
| 8 |
+
"num_samples_requested": 20000,
|
| 9 |
+
"num_samples_actual": 20000,
|
| 10 |
+
"batch_seqs": 32,
|
| 11 |
+
"stats_top_k": 100,
|
| 12 |
+
"stream_seed": 0,
|
| 13 |
+
"sparsity_thresholds": [
|
| 14 |
+
0.001,
|
| 15 |
+
0.01,
|
| 16 |
+
0.05,
|
| 17 |
+
0.1,
|
| 18 |
+
0.5,
|
| 19 |
+
1.0
|
| 20 |
+
],
|
| 21 |
+
"model_hf_name": "openai-community/gpt2",
|
| 22 |
+
"hook_target_name": "transformer.h.8",
|
| 23 |
+
"tokenizer_hf_name": "openai-community/gpt2",
|
| 24 |
+
"d_sae": 3072,
|
| 25 |
+
"d_in": 768
|
| 26 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 4,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 2,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 2.95,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 5,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/l0_per_token.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_samples": 20000,
|
| 3 |
+
"n_positions": 20480000,
|
| 4 |
+
"l0_per_token_0": 125.77726040039063,
|
| 5 |
+
"l0_per_token_0.001": 125.77724150390625,
|
| 6 |
+
"l0_per_token_0.01": 125.77699580078125,
|
| 7 |
+
"l0_per_token_0.05": 125.77617680664062,
|
| 8 |
+
"l0_per_token_0.1": 125.774969140625,
|
| 9 |
+
"l0_per_token_0.5": 125.6611578125,
|
| 10 |
+
"l0_per_token_1.0": 93.548204296875,
|
| 11 |
+
"l1_per_token": 390.5783639531069,
|
| 12 |
+
"r2_score": 0.9409605591020928,
|
| 13 |
+
"l2_loss": 1.384618591970846
|
| 14 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_128/seed2/stats.fineweb.split_train.seed_no/meta.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "fineweb",
|
| 3 |
+
"split": "train",
|
| 4 |
+
"hf_dataset": "HuggingFaceFW/fineweb",
|
| 5 |
+
"hf_dataset_config": null,
|
| 6 |
+
"hf_dataset_text_field": "text",
|
| 7 |
+
"context_length": 1024,
|
| 8 |
+
"num_samples_requested": 20000,
|
| 9 |
+
"num_samples_actual": 20000,
|
| 10 |
+
"batch_seqs": 32,
|
| 11 |
+
"stats_top_k": 100,
|
| 12 |
+
"stream_seed": 0,
|
| 13 |
+
"sparsity_thresholds": [
|
| 14 |
+
0.001,
|
| 15 |
+
0.01,
|
| 16 |
+
0.05,
|
| 17 |
+
0.1,
|
| 18 |
+
0.5,
|
| 19 |
+
1.0
|
| 20 |
+
],
|
| 21 |
+
"model_hf_name": "openai-community/gpt2",
|
| 22 |
+
"hook_target_name": "transformer.h.8",
|
| 23 |
+
"tokenizer_hf_name": "openai-community/gpt2",
|
| 24 |
+
"d_sae": 3072,
|
| 25 |
+
"d_in": 768
|
| 26 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_32/code_metrics_step_final.json
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": 1,
|
| 3 |
+
"step": "final",
|
| 4 |
+
"n_tokens": 600000,
|
| 5 |
+
"thresholds": [
|
| 6 |
+
0.5,
|
| 7 |
+
0.6,
|
| 8 |
+
0.7,
|
| 9 |
+
0.8,
|
| 10 |
+
0.9
|
| 11 |
+
],
|
| 12 |
+
"seeds": [
|
| 13 |
+
"seed0",
|
| 14 |
+
"seed1",
|
| 15 |
+
"seed2"
|
| 16 |
+
],
|
| 17 |
+
"per_seed": {
|
| 18 |
+
"seed0": {
|
| 19 |
+
"l0": 33.676136666666665,
|
| 20 |
+
"recon_r2": 0.8975673795678348,
|
| 21 |
+
"l1_per_token": 253.82649805907457
|
| 22 |
+
},
|
| 23 |
+
"seed1": {
|
| 24 |
+
"l0": 33.846435,
|
| 25 |
+
"recon_r2": 0.8977308049876321,
|
| 26 |
+
"l1_per_token": 254.91719231354833
|
| 27 |
+
},
|
| 28 |
+
"seed2": {
|
| 29 |
+
"l0": 33.651356666666665,
|
| 30 |
+
"recon_r2": 0.8976816471087172,
|
| 31 |
+
"l1_per_token": 255.21824300224597
|
| 32 |
+
}
|
| 33 |
+
},
|
| 34 |
+
"per_pair": {
|
| 35 |
+
"seed0\u2194seed1": {
|
| 36 |
+
"alive_source": "stats",
|
| 37 |
+
"mcc_signed": 0.8431015566494878,
|
| 38 |
+
"mcc_abs": 0.8431068267004274,
|
| 39 |
+
"act_iou": 0.5771575229163576,
|
| 40 |
+
"act_iou_n_pairs": 3072,
|
| 41 |
+
"input_r2": 0.9810264992004553,
|
| 42 |
+
"matched_mcc_signed": {
|
| 43 |
+
"0.6": 0.8933773801726083,
|
| 44 |
+
"0.7": 0.9059894971164321,
|
| 45 |
+
"0.8": 0.9244591009457132,
|
| 46 |
+
"0.9": 0.9450882109904006
|
| 47 |
+
},
|
| 48 |
+
"matched_mcc_abs": {
|
| 49 |
+
"0.6": 0.8941537238424455,
|
| 50 |
+
"0.7": 0.9051395484599205,
|
| 51 |
+
"0.8": 0.9227457160326656,
|
| 52 |
+
"0.9": 0.9450882109904014
|
| 53 |
+
}
|
| 54 |
+
},
|
| 55 |
+
"seed0\u2194seed2": {
|
| 56 |
+
"alive_source": "stats",
|
| 57 |
+
"mcc_signed": 0.8272049425533208,
|
| 58 |
+
"mcc_abs": 0.8272087362405648,
|
| 59 |
+
"act_iou": 0.5592969803272019,
|
| 60 |
+
"act_iou_n_pairs": 3072,
|
| 61 |
+
"input_r2": 0.9800477389075511,
|
| 62 |
+
"matched_mcc_signed": {
|
| 63 |
+
"0.6": 0.8880478491749985,
|
| 64 |
+
"0.7": 0.9024123006077128,
|
| 65 |
+
"0.8": 0.9184008541940856,
|
| 66 |
+
"0.9": 0.9426502364837515
|
| 67 |
+
},
|
| 68 |
+
"matched_mcc_abs": {
|
| 69 |
+
"0.6": 0.8882316570609861,
|
| 70 |
+
"0.7": 0.9017869623984147,
|
| 71 |
+
"0.8": 0.9181568606864884,
|
| 72 |
+
"0.9": 0.9426502364837515
|
| 73 |
+
}
|
| 74 |
+
},
|
| 75 |
+
"seed1\u2194seed2": {
|
| 76 |
+
"alive_source": "stats",
|
| 77 |
+
"mcc_signed": 0.8336904002024171,
|
| 78 |
+
"mcc_abs": 0.8337068280859405,
|
| 79 |
+
"act_iou": 0.5680103794159135,
|
| 80 |
+
"act_iou_n_pairs": 3072,
|
| 81 |
+
"input_r2": 0.9805106627129005,
|
| 82 |
+
"matched_mcc_signed": {
|
| 83 |
+
"0.6": 0.8906078080022163,
|
| 84 |
+
"0.7": 0.9041514540728112,
|
| 85 |
+
"0.8": 0.9187560352020746,
|
| 86 |
+
"0.9": 0.9430678425864627
|
| 87 |
+
},
|
| 88 |
+
"matched_mcc_abs": {
|
| 89 |
+
"0.6": 0.8897145007397163,
|
| 90 |
+
"0.7": 0.9036372232099904,
|
| 91 |
+
"0.8": 0.917934181590078,
|
| 92 |
+
"0.9": 0.9430055361585956
|
| 93 |
+
}
|
| 94 |
+
}
|
| 95 |
+
},
|
| 96 |
+
"summary": {
|
| 97 |
+
"n_seeds": 3,
|
| 98 |
+
"n_pairs": 3,
|
| 99 |
+
"report_band": [
|
| 100 |
+
0.6,
|
| 101 |
+
0.7,
|
| 102 |
+
0.8,
|
| 103 |
+
0.9
|
| 104 |
+
],
|
| 105 |
+
"l0_mean": 33.724642777777774,
|
| 106 |
+
"l0_std": 0.10620038203966387,
|
| 107 |
+
"recon_r2_mean": 0.8976599438880614,
|
| 108 |
+
"recon_r2_std": 8.384652228391995e-05,
|
| 109 |
+
"l1_per_token_mean": 254.65397779162296,
|
| 110 |
+
"l1_per_token_std": 0.7322567241924753,
|
| 111 |
+
"input_r2_mean": 0.9805283002736357,
|
| 112 |
+
"input_r2_std": 0.0004896184641156311,
|
| 113 |
+
"mcc_signed_mean": 0.8346656331350752,
|
| 114 |
+
"mcc_signed_std": 0.007993052882680055,
|
| 115 |
+
"mcc_abs_mean": 0.8346741303423109,
|
| 116 |
+
"mcc_abs_std": 0.007993064200222446,
|
| 117 |
+
"matched_mcc_signed_per_threshold": {
|
| 118 |
+
"0.6": 0.8906776791166077,
|
| 119 |
+
"0.7": 0.904184417265652,
|
| 120 |
+
"0.8": 0.9205386634472911,
|
| 121 |
+
"0.9": 0.9436020966868716
|
| 122 |
+
},
|
| 123 |
+
"matched_mcc_signed_band": 0.9147507141291056,
|
| 124 |
+
"matched_mcc_abs_per_threshold": {
|
| 125 |
+
"0.6": 0.890699960547716,
|
| 126 |
+
"0.7": 0.903521244689442,
|
| 127 |
+
"0.8": 0.919612252769744,
|
| 128 |
+
"0.9": 0.9435813278775829
|
| 129 |
+
},
|
| 130 |
+
"matched_mcc_abs_band": 0.9143536964711212,
|
| 131 |
+
"act_iou_mean": 0.568154960886491,
|
| 132 |
+
"act_iou_std": 0.008931149043991313,
|
| 133 |
+
"act_iou_n_pairs": 3072.0,
|
| 134 |
+
"alive_source": "stats"
|
| 135 |
+
}
|
| 136 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_32/dict_metrics_step_final.json
ADDED
|
@@ -0,0 +1,595 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": 3,
|
| 3 |
+
"step": "final",
|
| 4 |
+
"linkage": "complete",
|
| 5 |
+
"thresholds": [
|
| 6 |
+
0.5,
|
| 7 |
+
0.6,
|
| 8 |
+
0.7,
|
| 9 |
+
0.8,
|
| 10 |
+
0.9
|
| 11 |
+
],
|
| 12 |
+
"seeds": [
|
| 13 |
+
"seed0",
|
| 14 |
+
"seed1",
|
| 15 |
+
"seed2"
|
| 16 |
+
],
|
| 17 |
+
"per_seed": {
|
| 18 |
+
"seed0": {
|
| 19 |
+
"d_sae": 3072,
|
| 20 |
+
"d_in": 768,
|
| 21 |
+
"l0": 34,
|
| 22 |
+
"l0_raw": 34.167963525390626,
|
| 23 |
+
"recon_r2": 0.8977384601466455,
|
| 24 |
+
"l1_per_token": 255.04705028489641,
|
| 25 |
+
"stats_source": "stats",
|
| 26 |
+
"n_dead": 0,
|
| 27 |
+
"dead_rate": 0.0,
|
| 28 |
+
"alive_source": "stats",
|
| 29 |
+
"n_dense": 20,
|
| 30 |
+
"dense_rate": 0.006510416666666667,
|
| 31 |
+
"dense_threshold": 0.1,
|
| 32 |
+
"n_dup_signed": {
|
| 33 |
+
"0.5": 313,
|
| 34 |
+
"0.6": 0,
|
| 35 |
+
"0.7": 0,
|
| 36 |
+
"0.8": 0,
|
| 37 |
+
"0.9": 0
|
| 38 |
+
},
|
| 39 |
+
"n_dup_clusters_signed": {
|
| 40 |
+
"0.5": 149,
|
| 41 |
+
"0.6": 0,
|
| 42 |
+
"0.7": 0,
|
| 43 |
+
"0.8": 0,
|
| 44 |
+
"0.9": 0
|
| 45 |
+
},
|
| 46 |
+
"n_dup_removed_signed": {
|
| 47 |
+
"0.5": 164,
|
| 48 |
+
"0.6": 0,
|
| 49 |
+
"0.7": 0,
|
| 50 |
+
"0.8": 0,
|
| 51 |
+
"0.9": 0
|
| 52 |
+
},
|
| 53 |
+
"largest_dup_signed": {
|
| 54 |
+
"0.5": 4,
|
| 55 |
+
"0.6": 1,
|
| 56 |
+
"0.7": 1,
|
| 57 |
+
"0.8": 1,
|
| 58 |
+
"0.9": 1
|
| 59 |
+
},
|
| 60 |
+
"babel_mu1_signed": 0.37319397926330566,
|
| 61 |
+
"n_dup_abs": {
|
| 62 |
+
"0.5": 328,
|
| 63 |
+
"0.6": 40,
|
| 64 |
+
"0.7": 26,
|
| 65 |
+
"0.8": 8,
|
| 66 |
+
"0.9": 0
|
| 67 |
+
},
|
| 68 |
+
"n_dup_clusters_abs": {
|
| 69 |
+
"0.5": 155,
|
| 70 |
+
"0.6": 20,
|
| 71 |
+
"0.7": 13,
|
| 72 |
+
"0.8": 4,
|
| 73 |
+
"0.9": 0
|
| 74 |
+
},
|
| 75 |
+
"n_dup_removed_abs": {
|
| 76 |
+
"0.5": 173,
|
| 77 |
+
"0.6": 20,
|
| 78 |
+
"0.7": 13,
|
| 79 |
+
"0.8": 4,
|
| 80 |
+
"0.9": 0
|
| 81 |
+
},
|
| 82 |
+
"largest_dup_abs": {
|
| 83 |
+
"0.5": 4,
|
| 84 |
+
"0.6": 2,
|
| 85 |
+
"0.7": 2,
|
| 86 |
+
"0.8": 2,
|
| 87 |
+
"0.9": 1
|
| 88 |
+
},
|
| 89 |
+
"babel_mu1_abs": 0.37956738471984863,
|
| 90 |
+
"dup_excludes_dead": true
|
| 91 |
+
},
|
| 92 |
+
"seed1": {
|
| 93 |
+
"d_sae": 3072,
|
| 94 |
+
"d_in": 768,
|
| 95 |
+
"l0": 34,
|
| 96 |
+
"l0_raw": 34.20432915039063,
|
| 97 |
+
"recon_r2": 0.8978192650851368,
|
| 98 |
+
"l1_per_token": 255.75038515835834,
|
| 99 |
+
"stats_source": "stats",
|
| 100 |
+
"n_dead": 0,
|
| 101 |
+
"dead_rate": 0.0,
|
| 102 |
+
"alive_source": "stats",
|
| 103 |
+
"n_dense": 21,
|
| 104 |
+
"dense_rate": 0.0068359375,
|
| 105 |
+
"dense_threshold": 0.1,
|
| 106 |
+
"n_dup_signed": {
|
| 107 |
+
"0.5": 302,
|
| 108 |
+
"0.6": 0,
|
| 109 |
+
"0.7": 0,
|
| 110 |
+
"0.8": 0,
|
| 111 |
+
"0.9": 0
|
| 112 |
+
},
|
| 113 |
+
"n_dup_clusters_signed": {
|
| 114 |
+
"0.5": 146,
|
| 115 |
+
"0.6": 0,
|
| 116 |
+
"0.7": 0,
|
| 117 |
+
"0.8": 0,
|
| 118 |
+
"0.9": 0
|
| 119 |
+
},
|
| 120 |
+
"n_dup_removed_signed": {
|
| 121 |
+
"0.5": 156,
|
| 122 |
+
"0.6": 0,
|
| 123 |
+
"0.7": 0,
|
| 124 |
+
"0.8": 0,
|
| 125 |
+
"0.9": 0
|
| 126 |
+
},
|
| 127 |
+
"largest_dup_signed": {
|
| 128 |
+
"0.5": 4,
|
| 129 |
+
"0.6": 1,
|
| 130 |
+
"0.7": 1,
|
| 131 |
+
"0.8": 1,
|
| 132 |
+
"0.9": 1
|
| 133 |
+
},
|
| 134 |
+
"babel_mu1_signed": 0.37258124351501465,
|
| 135 |
+
"n_dup_abs": {
|
| 136 |
+
"0.5": 321,
|
| 137 |
+
"0.6": 36,
|
| 138 |
+
"0.7": 20,
|
| 139 |
+
"0.8": 4,
|
| 140 |
+
"0.9": 0
|
| 141 |
+
},
|
| 142 |
+
"n_dup_clusters_abs": {
|
| 143 |
+
"0.5": 155,
|
| 144 |
+
"0.6": 18,
|
| 145 |
+
"0.7": 10,
|
| 146 |
+
"0.8": 2,
|
| 147 |
+
"0.9": 0
|
| 148 |
+
},
|
| 149 |
+
"n_dup_removed_abs": {
|
| 150 |
+
"0.5": 166,
|
| 151 |
+
"0.6": 18,
|
| 152 |
+
"0.7": 10,
|
| 153 |
+
"0.8": 2,
|
| 154 |
+
"0.9": 0
|
| 155 |
+
},
|
| 156 |
+
"largest_dup_abs": {
|
| 157 |
+
"0.5": 4,
|
| 158 |
+
"0.6": 2,
|
| 159 |
+
"0.7": 2,
|
| 160 |
+
"0.8": 2,
|
| 161 |
+
"0.9": 1
|
| 162 |
+
},
|
| 163 |
+
"babel_mu1_abs": 0.3790799081325531,
|
| 164 |
+
"dup_excludes_dead": true
|
| 165 |
+
},
|
| 166 |
+
"seed2": {
|
| 167 |
+
"d_sae": 3072,
|
| 168 |
+
"d_in": 768,
|
| 169 |
+
"l0": 34,
|
| 170 |
+
"l0_raw": 34.06631484375,
|
| 171 |
+
"recon_r2": 0.8978286378898942,
|
| 172 |
+
"l1_per_token": 256.2946467344169,
|
| 173 |
+
"stats_source": "stats",
|
| 174 |
+
"n_dead": 0,
|
| 175 |
+
"dead_rate": 0.0,
|
| 176 |
+
"alive_source": "stats",
|
| 177 |
+
"n_dense": 20,
|
| 178 |
+
"dense_rate": 0.006510416666666667,
|
| 179 |
+
"dense_threshold": 0.1,
|
| 180 |
+
"n_dup_signed": {
|
| 181 |
+
"0.5": 288,
|
| 182 |
+
"0.6": 0,
|
| 183 |
+
"0.7": 0,
|
| 184 |
+
"0.8": 0,
|
| 185 |
+
"0.9": 0
|
| 186 |
+
},
|
| 187 |
+
"n_dup_clusters_signed": {
|
| 188 |
+
"0.5": 136,
|
| 189 |
+
"0.6": 0,
|
| 190 |
+
"0.7": 0,
|
| 191 |
+
"0.8": 0,
|
| 192 |
+
"0.9": 0
|
| 193 |
+
},
|
| 194 |
+
"n_dup_removed_signed": {
|
| 195 |
+
"0.5": 152,
|
| 196 |
+
"0.6": 0,
|
| 197 |
+
"0.7": 0,
|
| 198 |
+
"0.8": 0,
|
| 199 |
+
"0.9": 0
|
| 200 |
+
},
|
| 201 |
+
"largest_dup_signed": {
|
| 202 |
+
"0.5": 4,
|
| 203 |
+
"0.6": 1,
|
| 204 |
+
"0.7": 1,
|
| 205 |
+
"0.8": 1,
|
| 206 |
+
"0.9": 1
|
| 207 |
+
},
|
| 208 |
+
"babel_mu1_signed": 0.37247905135154724,
|
| 209 |
+
"n_dup_abs": {
|
| 210 |
+
"0.5": 305,
|
| 211 |
+
"0.6": 32,
|
| 212 |
+
"0.7": 18,
|
| 213 |
+
"0.8": 8,
|
| 214 |
+
"0.9": 0
|
| 215 |
+
},
|
| 216 |
+
"n_dup_clusters_abs": {
|
| 217 |
+
"0.5": 143,
|
| 218 |
+
"0.6": 16,
|
| 219 |
+
"0.7": 9,
|
| 220 |
+
"0.8": 4,
|
| 221 |
+
"0.9": 0
|
| 222 |
+
},
|
| 223 |
+
"n_dup_removed_abs": {
|
| 224 |
+
"0.5": 162,
|
| 225 |
+
"0.6": 16,
|
| 226 |
+
"0.7": 9,
|
| 227 |
+
"0.8": 4,
|
| 228 |
+
"0.9": 0
|
| 229 |
+
},
|
| 230 |
+
"largest_dup_abs": {
|
| 231 |
+
"0.5": 4,
|
| 232 |
+
"0.6": 2,
|
| 233 |
+
"0.7": 2,
|
| 234 |
+
"0.8": 2,
|
| 235 |
+
"0.9": 1
|
| 236 |
+
},
|
| 237 |
+
"babel_mu1_abs": 0.37893176078796387,
|
| 238 |
+
"dup_excludes_dead": true
|
| 239 |
+
}
|
| 240 |
+
},
|
| 241 |
+
"per_pair": {
|
| 242 |
+
"seed0\u2194seed1": {
|
| 243 |
+
"dcs_signed": 0.8927777409553528,
|
| 244 |
+
"dcs_abs": 0.8932483196258545,
|
| 245 |
+
"frac_sign_flipped": 0.009440104166666666,
|
| 246 |
+
"alive_source": "stats",
|
| 247 |
+
"frac_denominator": "d_sae",
|
| 248 |
+
"n_alive_a": 3072,
|
| 249 |
+
"n_alive_b": 3072,
|
| 250 |
+
"matched_cos_signed": {
|
| 251 |
+
"0.5": 0.8908541202545166,
|
| 252 |
+
"0.6": 0.8927777409553528,
|
| 253 |
+
"0.7": 0.8927777409553528,
|
| 254 |
+
"0.8": 0.8927777409553528,
|
| 255 |
+
"0.9": 0.8927777409553528
|
| 256 |
+
},
|
| 257 |
+
"unique_frac_signed": {
|
| 258 |
+
"0.5": 0.9479166666666666,
|
| 259 |
+
"0.6": 1.0,
|
| 260 |
+
"0.7": 1.0,
|
| 261 |
+
"0.8": 1.0,
|
| 262 |
+
"0.9": 1.0
|
| 263 |
+
},
|
| 264 |
+
"dedup_dcs_signed": {
|
| 265 |
+
"0.5": 0.8444554681579272,
|
| 266 |
+
"0.6": 0.8927777409553528,
|
| 267 |
+
"0.7": 0.8927777409553528,
|
| 268 |
+
"0.8": 0.8927777409553528,
|
| 269 |
+
"0.9": 0.8927777409553528
|
| 270 |
+
},
|
| 271 |
+
"matched_frac_signed": {
|
| 272 |
+
"0.5": 0.8863932291666666,
|
| 273 |
+
"0.6": 0.9300130208333334,
|
| 274 |
+
"0.7": 0.8990885416666666,
|
| 275 |
+
"0.8": 0.8447265625,
|
| 276 |
+
"0.9": 0.75
|
| 277 |
+
},
|
| 278 |
+
"matched_cos_abs": {
|
| 279 |
+
"0.5": 0.8912460207939148,
|
| 280 |
+
"0.6": 0.8940423130989075,
|
| 281 |
+
"0.7": 0.8936648368835449,
|
| 282 |
+
"0.8": 0.8935339450836182,
|
| 283 |
+
"0.9": 0.8932483792304993
|
| 284 |
+
},
|
| 285 |
+
"unique_frac_abs": {
|
| 286 |
+
"0.5": 0.94482421875,
|
| 287 |
+
"0.6": 0.9938151041666666,
|
| 288 |
+
"0.7": 0.9962565104166666,
|
| 289 |
+
"0.8": 0.9990234375,
|
| 290 |
+
"0.9": 1.0
|
| 291 |
+
},
|
| 292 |
+
"dedup_dcs_abs": {
|
| 293 |
+
"0.5": 0.8420708253106568,
|
| 294 |
+
"0.6": 0.8885127545217983,
|
| 295 |
+
"0.7": 0.8903194118756801,
|
| 296 |
+
"0.8": 0.8926613533403724,
|
| 297 |
+
"0.9": 0.8932483792304993
|
| 298 |
+
},
|
| 299 |
+
"matched_frac_abs": {
|
| 300 |
+
"0.5": 0.8834635416666666,
|
| 301 |
+
"0.6": 0.9231770833333334,
|
| 302 |
+
"0.7": 0.8955078125,
|
| 303 |
+
"0.8": 0.8470052083333334,
|
| 304 |
+
"0.9": 0.75
|
| 305 |
+
}
|
| 306 |
+
},
|
| 307 |
+
"seed0\u2194seed2": {
|
| 308 |
+
"dcs_signed": 0.882415771484375,
|
| 309 |
+
"dcs_abs": 0.8828290104866028,
|
| 310 |
+
"frac_sign_flipped": 0.0078125,
|
| 311 |
+
"alive_source": "stats",
|
| 312 |
+
"frac_denominator": "d_sae",
|
| 313 |
+
"n_alive_a": 3072,
|
| 314 |
+
"n_alive_b": 3072,
|
| 315 |
+
"matched_cos_signed": {
|
| 316 |
+
"0.5": 0.881562352180481,
|
| 317 |
+
"0.6": 0.882415771484375,
|
| 318 |
+
"0.7": 0.882415771484375,
|
| 319 |
+
"0.8": 0.882415771484375,
|
| 320 |
+
"0.9": 0.882415771484375
|
| 321 |
+
},
|
| 322 |
+
"unique_frac_signed": {
|
| 323 |
+
"0.5": 0.9485677083333334,
|
| 324 |
+
"0.6": 1.0,
|
| 325 |
+
"0.7": 1.0,
|
| 326 |
+
"0.8": 1.0,
|
| 327 |
+
"0.9": 1.0
|
| 328 |
+
},
|
| 329 |
+
"dedup_dcs_signed": {
|
| 330 |
+
"0.5": 0.8362215801607817,
|
| 331 |
+
"0.6": 0.882415771484375,
|
| 332 |
+
"0.7": 0.882415771484375,
|
| 333 |
+
"0.8": 0.882415771484375,
|
| 334 |
+
"0.9": 0.882415771484375
|
| 335 |
+
},
|
| 336 |
+
"matched_frac_signed": {
|
| 337 |
+
"0.5": 0.8798828125,
|
| 338 |
+
"0.6": 0.9108072916666666,
|
| 339 |
+
"0.7": 0.8782552083333334,
|
| 340 |
+
"0.8": 0.8317057291666666,
|
| 341 |
+
"0.9": 0.7233072916666666
|
| 342 |
+
},
|
| 343 |
+
"matched_cos_abs": {
|
| 344 |
+
"0.5": 0.8822004199028015,
|
| 345 |
+
"0.6": 0.8837887644767761,
|
| 346 |
+
"0.7": 0.8835625648498535,
|
| 347 |
+
"0.8": 0.8828677535057068,
|
| 348 |
+
"0.9": 0.8828290104866028
|
| 349 |
+
},
|
| 350 |
+
"unique_frac_abs": {
|
| 351 |
+
"0.5": 0.9454752604166666,
|
| 352 |
+
"0.6": 0.994140625,
|
| 353 |
+
"0.7": 0.9964192708333334,
|
| 354 |
+
"0.8": 0.9986979166666666,
|
| 355 |
+
"0.9": 1.0
|
| 356 |
+
},
|
| 357 |
+
"dedup_dcs_abs": {
|
| 358 |
+
"0.5": 0.8340986717472939,
|
| 359 |
+
"0.6": 0.87861031468492,
|
| 360 |
+
"0.7": 0.8803987666033208,
|
| 361 |
+
"0.8": 0.8817181861183295,
|
| 362 |
+
"0.9": 0.8828290104866028
|
| 363 |
+
},
|
| 364 |
+
"matched_frac_abs": {
|
| 365 |
+
"0.5": 0.876953125,
|
| 366 |
+
"0.6": 0.9049479166666666,
|
| 367 |
+
"0.7": 0.8753255208333334,
|
| 368 |
+
"0.8": 0.8304036458333334,
|
| 369 |
+
"0.9": 0.7233072916666666
|
| 370 |
+
}
|
| 371 |
+
},
|
| 372 |
+
"seed1\u2194seed2": {
|
| 373 |
+
"dcs_signed": 0.8868472576141357,
|
| 374 |
+
"dcs_abs": 0.8873586058616638,
|
| 375 |
+
"frac_sign_flipped": 0.008463541666666666,
|
| 376 |
+
"alive_source": "stats",
|
| 377 |
+
"frac_denominator": "d_sae",
|
| 378 |
+
"n_alive_a": 3072,
|
| 379 |
+
"n_alive_b": 3072,
|
| 380 |
+
"matched_cos_signed": {
|
| 381 |
+
"0.5": 0.884628176689148,
|
| 382 |
+
"0.6": 0.8868473172187805,
|
| 383 |
+
"0.7": 0.8868473172187805,
|
| 384 |
+
"0.8": 0.8868473172187805,
|
| 385 |
+
"0.9": 0.8868473172187805
|
| 386 |
+
},
|
| 387 |
+
"unique_frac_signed": {
|
| 388 |
+
"0.5": 0.9498697916666666,
|
| 389 |
+
"0.6": 1.0,
|
| 390 |
+
"0.7": 1.0,
|
| 391 |
+
"0.8": 1.0,
|
| 392 |
+
"0.9": 1.0
|
| 393 |
+
},
|
| 394 |
+
"dedup_dcs_signed": {
|
| 395 |
+
"0.5": 0.8402815818941841,
|
| 396 |
+
"0.6": 0.8868473172187805,
|
| 397 |
+
"0.7": 0.8868473172187805,
|
| 398 |
+
"0.8": 0.8868473172187805,
|
| 399 |
+
"0.9": 0.8868473172187805
|
| 400 |
+
},
|
| 401 |
+
"matched_frac_signed": {
|
| 402 |
+
"0.5": 0.8815104166666666,
|
| 403 |
+
"0.6": 0.9163411458333334,
|
| 404 |
+
"0.7": 0.8857421875,
|
| 405 |
+
"0.8": 0.841796875,
|
| 406 |
+
"0.9": 0.7317708333333334
|
| 407 |
+
},
|
| 408 |
+
"matched_cos_abs": {
|
| 409 |
+
"0.5": 0.8856135010719299,
|
| 410 |
+
"0.6": 0.8882299661636353,
|
| 411 |
+
"0.7": 0.8876924514770508,
|
| 412 |
+
"0.8": 0.8877029418945312,
|
| 413 |
+
"0.9": 0.8873586058616638
|
| 414 |
+
},
|
| 415 |
+
"unique_frac_abs": {
|
| 416 |
+
"0.5": 0.9466145833333334,
|
| 417 |
+
"0.6": 0.9944661458333334,
|
| 418 |
+
"0.7": 0.9969075520833334,
|
| 419 |
+
"0.8": 0.9990234375,
|
| 420 |
+
"0.9": 1.0
|
| 421 |
+
},
|
| 422 |
+
"dedup_dcs_abs": {
|
| 423 |
+
"0.5": 0.8383346553115796,
|
| 424 |
+
"0.6": 0.8833146310644224,
|
| 425 |
+
"0.7": 0.8849473088048398,
|
| 426 |
+
"0.8": 0.8868360444903374,
|
| 427 |
+
"0.9": 0.8873586058616638
|
| 428 |
+
},
|
| 429 |
+
"matched_frac_abs": {
|
| 430 |
+
"0.5": 0.8785807291666666,
|
| 431 |
+
"0.6": 0.9108072916666666,
|
| 432 |
+
"0.7": 0.8834635416666666,
|
| 433 |
+
"0.8": 0.841796875,
|
| 434 |
+
"0.9": 0.7320963541666666
|
| 435 |
+
}
|
| 436 |
+
}
|
| 437 |
+
},
|
| 438 |
+
"summary": {
|
| 439 |
+
"n_seeds": 3,
|
| 440 |
+
"n_pairs": 3,
|
| 441 |
+
"report_band": [
|
| 442 |
+
0.6,
|
| 443 |
+
0.7,
|
| 444 |
+
0.8,
|
| 445 |
+
0.9
|
| 446 |
+
],
|
| 447 |
+
"n_dead_mean": 0,
|
| 448 |
+
"n_dead_std": 0.0,
|
| 449 |
+
"n_dense_mean": 20,
|
| 450 |
+
"n_dense_std": 0.5773502691896258,
|
| 451 |
+
"l0_mean": 34,
|
| 452 |
+
"l0_std": 0.07153421325772635,
|
| 453 |
+
"recon_r2_mean": 0.8977954543738922,
|
| 454 |
+
"recon_r2_std": 4.9580427810158145e-05,
|
| 455 |
+
"l1_per_token_mean": 255.69736072589058,
|
| 456 |
+
"l1_per_token_std": 0.6254861453644979,
|
| 457 |
+
"babel_mu1_signed_mean": 0.37275142470995587,
|
| 458 |
+
"babel_mu1_signed_std": 0.00038665450868511356,
|
| 459 |
+
"n_dup_signed_per_threshold": {
|
| 460 |
+
"0.5": 301.0,
|
| 461 |
+
"0.6": 0.0,
|
| 462 |
+
"0.7": 0.0,
|
| 463 |
+
"0.8": 0.0,
|
| 464 |
+
"0.9": 0.0
|
| 465 |
+
},
|
| 466 |
+
"n_dup_signed_band": 0.0,
|
| 467 |
+
"n_dup_signed_band_std": 0.0,
|
| 468 |
+
"n_dup_clusters_signed_per_threshold": {
|
| 469 |
+
"0.5": 143.66666666666666,
|
| 470 |
+
"0.6": 0.0,
|
| 471 |
+
"0.7": 0.0,
|
| 472 |
+
"0.8": 0.0,
|
| 473 |
+
"0.9": 0.0
|
| 474 |
+
},
|
| 475 |
+
"n_dup_clusters_signed_band": 0.0,
|
| 476 |
+
"n_dup_clusters_signed_band_std": 0.0,
|
| 477 |
+
"largest_dup_signed_per_threshold": {
|
| 478 |
+
"0.5": 4.0,
|
| 479 |
+
"0.6": 1.0,
|
| 480 |
+
"0.7": 1.0,
|
| 481 |
+
"0.8": 1.0,
|
| 482 |
+
"0.9": 1.0
|
| 483 |
+
},
|
| 484 |
+
"largest_dup_signed_band": 1.0,
|
| 485 |
+
"largest_dup_signed_band_std": 0.0,
|
| 486 |
+
"dcs_signed_mean": 0.8873469233512878,
|
| 487 |
+
"dcs_signed_std": 0.005199024159977689,
|
| 488 |
+
"dedup_dcs_signed_per_threshold": {
|
| 489 |
+
"0.5": 0.8403195434042976,
|
| 490 |
+
"0.6": 0.8873469432195028,
|
| 491 |
+
"0.7": 0.8873469432195028,
|
| 492 |
+
"0.8": 0.8873469432195028,
|
| 493 |
+
"0.9": 0.8873469432195028
|
| 494 |
+
},
|
| 495 |
+
"dedup_dcs_signed_band": 0.8873469432195028,
|
| 496 |
+
"dedup_dcs_signed_band_std": 0.005199021295861095,
|
| 497 |
+
"matched_cos_signed_per_threshold": {
|
| 498 |
+
"0.5": 0.8856815497080485,
|
| 499 |
+
"0.6": 0.8873469432195028,
|
| 500 |
+
"0.7": 0.8873469432195028,
|
| 501 |
+
"0.8": 0.8873469432195028,
|
| 502 |
+
"0.9": 0.8873469432195028
|
| 503 |
+
},
|
| 504 |
+
"matched_cos_signed_band": 0.8873469432195028,
|
| 505 |
+
"matched_cos_signed_band_std": 0.005199021295861095,
|
| 506 |
+
"unique_frac_signed_per_threshold": {
|
| 507 |
+
"0.5": 0.9487847222222222,
|
| 508 |
+
"0.6": 1.0,
|
| 509 |
+
"0.7": 1.0,
|
| 510 |
+
"0.8": 1.0,
|
| 511 |
+
"0.9": 1.0
|
| 512 |
+
},
|
| 513 |
+
"unique_frac_signed_band": 1.0,
|
| 514 |
+
"unique_frac_signed_band_std": 0.0,
|
| 515 |
+
"matched_frac_signed_per_threshold": {
|
| 516 |
+
"0.5": 0.882595486111111,
|
| 517 |
+
"0.6": 0.9190538194444445,
|
| 518 |
+
"0.7": 0.8876953125,
|
| 519 |
+
"0.8": 0.8394097222222222,
|
| 520 |
+
"0.9": 0.7350260416666666
|
| 521 |
+
},
|
| 522 |
+
"matched_frac_signed_band": 0.8452962239583334,
|
| 523 |
+
"matched_frac_signed_band_std": 0.010040813974395683,
|
| 524 |
+
"babel_mu1_abs_mean": 0.3791930178801219,
|
| 525 |
+
"babel_mu1_abs_std": 0.00033256549271153314,
|
| 526 |
+
"n_dup_abs_per_threshold": {
|
| 527 |
+
"0.5": 318.0,
|
| 528 |
+
"0.6": 36.0,
|
| 529 |
+
"0.7": 21.333333333333332,
|
| 530 |
+
"0.8": 6.666666666666667,
|
| 531 |
+
"0.9": 0.0
|
| 532 |
+
},
|
| 533 |
+
"n_dup_abs_band": 15.999999999999998,
|
| 534 |
+
"n_dup_abs_band_std": 2.179449471770337,
|
| 535 |
+
"n_dup_clusters_abs_per_threshold": {
|
| 536 |
+
"0.5": 151.0,
|
| 537 |
+
"0.6": 18.0,
|
| 538 |
+
"0.7": 10.666666666666666,
|
| 539 |
+
"0.8": 3.3333333333333335,
|
| 540 |
+
"0.9": 0.0
|
| 541 |
+
},
|
| 542 |
+
"n_dup_clusters_abs_band": 7.999999999999999,
|
| 543 |
+
"n_dup_clusters_abs_band_std": 1.0897247358851685,
|
| 544 |
+
"largest_dup_abs_per_threshold": {
|
| 545 |
+
"0.5": 4.0,
|
| 546 |
+
"0.6": 2.0,
|
| 547 |
+
"0.7": 2.0,
|
| 548 |
+
"0.8": 2.0,
|
| 549 |
+
"0.9": 1.0
|
| 550 |
+
},
|
| 551 |
+
"largest_dup_abs_band": 1.75,
|
| 552 |
+
"largest_dup_abs_band_std": 0.0,
|
| 553 |
+
"dcs_abs_mean": 0.8878119786580404,
|
| 554 |
+
"dcs_abs_std": 0.005224429241954978,
|
| 555 |
+
"dedup_dcs_abs_per_threshold": {
|
| 556 |
+
"0.5": 0.8381680507898434,
|
| 557 |
+
"0.6": 0.8834792334237136,
|
| 558 |
+
"0.7": 0.8852218290946136,
|
| 559 |
+
"0.8": 0.8870718613163464,
|
| 560 |
+
"0.9": 0.8878119985262553
|
| 561 |
+
},
|
| 562 |
+
"dedup_dcs_abs_band": 0.8858962305902321,
|
| 563 |
+
"dedup_dcs_abs_band_std": 0.005153995391320634,
|
| 564 |
+
"matched_cos_abs_per_threshold": {
|
| 565 |
+
"0.5": 0.8863533139228821,
|
| 566 |
+
"0.6": 0.888687014579773,
|
| 567 |
+
"0.7": 0.8883066177368164,
|
| 568 |
+
"0.8": 0.8880348801612854,
|
| 569 |
+
"0.9": 0.8878119985262553
|
| 570 |
+
},
|
| 571 |
+
"matched_cos_abs_band": 0.8882101277510325,
|
| 572 |
+
"matched_cos_abs_band_std": 0.005195743964577689,
|
| 573 |
+
"unique_frac_abs_per_threshold": {
|
| 574 |
+
"0.5": 0.9456380208333334,
|
| 575 |
+
"0.6": 0.994140625,
|
| 576 |
+
"0.7": 0.9965277777777778,
|
| 577 |
+
"0.8": 0.9989149305555555,
|
| 578 |
+
"0.9": 1.0
|
| 579 |
+
},
|
| 580 |
+
"unique_frac_abs_band": 0.9973958333333333,
|
| 581 |
+
"unique_frac_abs_band_std": 0.00017736405206471122,
|
| 582 |
+
"matched_frac_abs_per_threshold": {
|
| 583 |
+
"0.5": 0.879665798611111,
|
| 584 |
+
"0.6": 0.9129774305555555,
|
| 585 |
+
"0.7": 0.884765625,
|
| 586 |
+
"0.8": 0.8397352430555557,
|
| 587 |
+
"0.9": 0.735134548611111
|
| 588 |
+
},
|
| 589 |
+
"matched_frac_abs_band": 0.8431532118055556,
|
| 590 |
+
"matched_frac_abs_band_std": 0.01025853397418673,
|
| 591 |
+
"alive_source": "stats",
|
| 592 |
+
"frac_denominator": "d_sae",
|
| 593 |
+
"d_sae": 3072
|
| 594 |
+
}
|
| 595 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 4,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 0,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 7.1,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 15,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/l0_per_token.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_samples": 20000,
|
| 3 |
+
"n_positions": 20480000,
|
| 4 |
+
"l0_per_token_0": 34.167963525390626,
|
| 5 |
+
"l0_per_token_0.001": 34.167961572265625,
|
| 6 |
+
"l0_per_token_0.01": 34.167939111328124,
|
| 7 |
+
"l0_per_token_0.05": 34.167740185546876,
|
| 8 |
+
"l0_per_token_0.1": 34.16754931640625,
|
| 9 |
+
"l0_per_token_0.5": 34.164538037109374,
|
| 10 |
+
"l0_per_token_1.0": 34.15486186523437,
|
| 11 |
+
"l1_per_token": 255.04705028489641,
|
| 12 |
+
"r2_score": 0.8977384601466455,
|
| 13 |
+
"l2_loss": 2.3982820157353704
|
| 14 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_32/seed0/stats.fineweb.split_train.seed_no/meta.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "fineweb",
|
| 3 |
+
"split": "train",
|
| 4 |
+
"hf_dataset": "HuggingFaceFW/fineweb",
|
| 5 |
+
"hf_dataset_config": null,
|
| 6 |
+
"hf_dataset_text_field": "text",
|
| 7 |
+
"context_length": 1024,
|
| 8 |
+
"num_samples_requested": 20000,
|
| 9 |
+
"num_samples_actual": 20000,
|
| 10 |
+
"batch_seqs": 32,
|
| 11 |
+
"stats_top_k": 100,
|
| 12 |
+
"stream_seed": 0,
|
| 13 |
+
"sparsity_thresholds": [
|
| 14 |
+
0.001,
|
| 15 |
+
0.01,
|
| 16 |
+
0.05,
|
| 17 |
+
0.1,
|
| 18 |
+
0.5,
|
| 19 |
+
1.0
|
| 20 |
+
],
|
| 21 |
+
"model_hf_name": "openai-community/gpt2",
|
| 22 |
+
"hook_target_name": "transformer.h.8",
|
| 23 |
+
"tokenizer_hf_name": "openai-community/gpt2",
|
| 24 |
+
"d_sae": 3072,
|
| 25 |
+
"d_in": 768
|
| 26 |
+
}
|
lm/gpt2-small_layer8/consdl/x4/l0_32/seed1/config.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"d_in": 768,
|
| 3 |
+
"d_sae_factor": 4,
|
| 4 |
+
"architecture": "fista",
|
| 5 |
+
"activation_name": "relu",
|
| 6 |
+
"device": "cpu",
|
| 7 |
+
"seed": 1,
|
| 8 |
+
"fista_t_iters": 30,
|
| 9 |
+
"fista_step": 0.0,
|
| 10 |
+
"fista_lam": 7.1,
|
| 11 |
+
"fista_nonneg": true,
|
| 12 |
+
"fista_power_iters": 32,
|
| 13 |
+
"fista_r2_tol": 0.0001,
|
| 14 |
+
"fista_check_every": 3,
|
| 15 |
+
"fista_use_bf16": false,
|
| 16 |
+
"fista_step_cache_every": 50,
|
| 17 |
+
"fista_compile": false,
|
| 18 |
+
"fista_topk_k": 0,
|
| 19 |
+
"refit": true,
|
| 20 |
+
"refit_iters": 15,
|
| 21 |
+
"refit_thresh": 1e-08,
|
| 22 |
+
"class_type": "FistaSAE",
|
| 23 |
+
"module_name": "consdl.models.fista"
|
| 24 |
+
}
|