Hugging Face's logo

AwesomeInterpretability
/
gemma-sae

Model card Files
xet
Community
AwesomeInterpretability commited on
Commit
c32db47
·
verified ·
1 Parent(s): 06dd179

Upload folder using huggingface_hub

Browse files
Files changed (12) hide show
  1. trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_0/ae.pt +3 -0
  2. trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_0/config.json +32 -0
  3. trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_0/eval_results.json +1 -0
  4. trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_1/ae.pt +3 -0
  5. trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_1/config.json +32 -0
  6. trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_1/eval_results.json +1 -0
  7. trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_2/ae.pt +3 -0
  8. trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_2/config.json +32 -0
  9. trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_2/eval_results.json +1 -0
  10. trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_3/ae.pt +3 -0
  11. trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_3/config.json +32 -0
  12. trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_3/eval_results.json +1 -0
trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_0/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5b25aa2b33c17d3f5cb09fe0996d6ef6084f38d35ba171e6051d3d512311b5e
3
+ size 302066710
trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_0/config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "BatchTopKTrainer",
4
+ "dict_class": "BatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 244140,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 195312,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 3407,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "k": 50,
17
+ "device": "cuda:1",
18
+ "layer": 12,
19
+ "lm_name": "google/gemma-2-2b",
20
+ "wandb_name": "BatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_0",
21
+ "submodule_name": "resid_post_layer_12"
22
+ },
23
+ "buffer": {
24
+ "d_submodule": 2304,
25
+ "io": "out",
26
+ "n_ctxs": 122,
27
+ "ctx_len": 2048,
28
+ "refresh_batch_size": 4,
29
+ "out_batch_size": 2048,
30
+ "device": "cuda:1"
31
+ }
32
+ }
trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_0/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 72.7625, "l1_loss": 504.31, "l0": 71.59681884765625, "frac_variance_explained": 0.72431640625, "cossim": 0.85302734375, "l2_ratio": 0.8634765625, "relative_reconstruction_bias": 1.0107421875, "frac_alive": 0.95574951171875, "hyperparameters": {"n_inputs": 200, "context_length": 2048}}
trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_1/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:536fb09722a10a7d2ae65dc0abb74cd08ef251bc1034078c768fc4293ba9118c
3
+ size 302066710
trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_1/config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "BatchTopKTrainer",
4
+ "dict_class": "BatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 244140,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 195312,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 3407,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "k": 320,
17
+ "device": "cuda:1",
18
+ "layer": 12,
19
+ "lm_name": "google/gemma-2-2b",
20
+ "wandb_name": "BatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_1",
21
+ "submodule_name": "resid_post_layer_12"
22
+ },
23
+ "buffer": {
24
+ "d_submodule": 2304,
25
+ "io": "out",
26
+ "n_ctxs": 122,
27
+ "ctx_len": 2048,
28
+ "refresh_batch_size": 4,
29
+ "out_batch_size": 2048,
30
+ "device": "cuda:1"
31
+ }
32
+ }
trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_1/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 51.96875, "l1_loss": 1797.32, "l0": 459.0106689453125, "frac_variance_explained": 0.8615234375, "cossim": 0.92580078125, "l2_ratio": 0.92578125, "relative_reconstruction_bias": 0.9995703125, "frac_alive": 0.86260986328125, "hyperparameters": {"n_inputs": 200, "context_length": 2048}}
trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_2/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:736872bd8ab43656b7a2b3b80383d7f848a62a6332a995f914c0d1df47ee8093
3
+ size 302066710
trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_2/config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "BatchTopKTrainer",
4
+ "dict_class": "BatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 244140,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 195312,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 3407,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "k": 520,
17
+ "device": "cuda:1",
18
+ "layer": 12,
19
+ "lm_name": "google/gemma-2-2b",
20
+ "wandb_name": "BatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_2",
21
+ "submodule_name": "resid_post_layer_12"
22
+ },
23
+ "buffer": {
24
+ "d_submodule": 2304,
25
+ "io": "out",
26
+ "n_ctxs": 122,
27
+ "ctx_len": 2048,
28
+ "refresh_batch_size": 4,
29
+ "out_batch_size": 2048,
30
+ "device": "cuda:1"
31
+ }
32
+ }
trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_2/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 45.09, "l1_loss": 2697.04, "l0": 707.0885473632812, "frac_variance_explained": 0.8932421875, "cossim": 0.94154296875, "l2_ratio": 0.9376171875, "relative_reconstruction_bias": 0.99638671875, "frac_alive": 0.73834228515625, "hyperparameters": {"n_inputs": 200, "context_length": 2048}}
trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_3/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d06fa4e7653ccd1e4d9ba0acfcde6814b19700ae26636905be003ddb56941b2
3
+ size 302066710
trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_3/config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "BatchTopKTrainer",
4
+ "dict_class": "BatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 244140,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 195312,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 3407,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "k": 820,
17
+ "device": "cuda:1",
18
+ "layer": 12,
19
+ "lm_name": "google/gemma-2-2b",
20
+ "wandb_name": "BatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_3",
21
+ "submodule_name": "resid_post_layer_12"
22
+ },
23
+ "buffer": {
24
+ "d_submodule": 2304,
25
+ "io": "out",
26
+ "n_ctxs": 122,
27
+ "ctx_len": 2048,
28
+ "refresh_batch_size": 4,
29
+ "out_batch_size": 2048,
30
+ "device": "cuda:1"
31
+ }
32
+ }
trained_saes_2__google_gemma-2-2b_batch_top_k/resid_post_layer_12/trainer_3/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 36.25375, "l1_loss": 4206.24, "l0": 1072.57626953125, "frac_variance_explained": 0.92908203125, "cossim": 0.96064453125, "l2_ratio": 0.94982421875, "relative_reconstruction_bias": 0.99130859375, "frac_alive": 0.50628662109375, "hyperparameters": {"n_inputs": 200, "context_length": 2048}}