Hugging Face's logo

AntonKorznikov
/
skip

Model card Files
xet
Community
AntonKorznikov commited on
Commit
bdbfbcb
·
verified ·
1 Parent(s): ec9a91d

Upload folder using huggingface_hub

Browse files
Files changed (15) hide show
  1. skip/ort_16/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/ae.pt +3 -0
  2. skip/ort_16/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/config.json +32 -0
  3. skip/ort_16/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/eval_results.json +1 -0
  4. skip/ort_32/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/ae.pt +3 -0
  5. skip/ort_32/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/config.json +32 -0
  6. skip/ort_32/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/eval_results.json +1 -0
  7. skip/ort_4/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/ae.pt +3 -0
  8. skip/ort_4/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/config.json +32 -0
  9. skip/ort_4/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/eval_results.json +1 -0
  10. skip/ort_64/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/ae.pt +3 -0
  11. skip/ort_64/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/config.json +32 -0
  12. skip/ort_64/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/eval_results.json +1 -0
  13. skip/ort_8/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/ae.pt +3 -0
  14. skip/ort_8/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/config.json +32 -0
  15. skip/ort_8/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/eval_results.json +1 -0
skip/ort_16/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2b16f10f2a4a7c52238e64aaf98517bd2443c202303b8b1415fc14a7622996c5
3
+ size 1208232982
skip/ort_16/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "BatchTopKTrainer",
4
+ "dict_class": "BatchTopKSAE",
5
+ "lr": 0.0002,
6
+ "steps": 244140,
7
+ "auxk_alpha": 0.0625,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 195312,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 65536,
16
+ "k": 70,
17
+ "device": "cuda:3",
18
+ "layer": 12,
19
+ "lm_name": "google/gemma-2-2b",
20
+ "wandb_name": "BatchTopKOrtTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_0",
21
+ "submodule_name": "resid_post_layer_12"
22
+ },
23
+ "buffer": {
24
+ "d_submodule": 2304,
25
+ "io": "out",
26
+ "n_ctxs": 244,
27
+ "ctx_len": 1024,
28
+ "refresh_batch_size": 32,
29
+ "out_batch_size": 2048,
30
+ "device": "cuda:3"
31
+ }
32
+ }
skip/ort_16/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 51.174698795180724, "l1_loss": 500.5903614457831, "l0": 69.4124062779438, "frac_variance_explained": 0.8085466867469879, "cossim": 0.93359375, "l2_ratio": 0.93359375, "relative_reconstruction_bias": 1.0000941265060241, "loss_original": 2.2974868222891565, "loss_reconstructed": 2.780308734939759, "loss_zero": 12.4375, "frac_recovered": 0.9525602409638554, "frac_alive": 0.98974609375, "hyperparameters": {"n_inputs": 1000, "context_length": 1024}}
skip/ort_32/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2f6919bd3fd59cb414a6f39fc05496bb7d4c027b5ba401110abb4fe0d35592cc
3
+ size 1208232982
skip/ort_32/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "BatchTopKTrainer",
4
+ "dict_class": "BatchTopKSAE",
5
+ "lr": 0.0002,
6
+ "steps": 244140,
7
+ "auxk_alpha": 0.0625,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 195312,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 65536,
16
+ "k": 70,
17
+ "device": "cuda:4",
18
+ "layer": 12,
19
+ "lm_name": "google/gemma-2-2b",
20
+ "wandb_name": "BatchTopKOrtTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_0",
21
+ "submodule_name": "resid_post_layer_12"
22
+ },
23
+ "buffer": {
24
+ "d_submodule": 2304,
25
+ "io": "out",
26
+ "n_ctxs": 244,
27
+ "ctx_len": 1024,
28
+ "refresh_batch_size": 32,
29
+ "out_batch_size": 2048,
30
+ "device": "cuda:4"
31
+ }
32
+ }
skip/ort_32/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 51.03012048192771, "l1_loss": 498.5542168674699, "l0": 69.87831644265049, "frac_variance_explained": 0.8097467996987951, "cossim": 0.93359375, "l2_ratio": 0.93359375, "relative_reconstruction_bias": 1.0007530120481927, "loss_original": 2.2974868222891565, "loss_reconstructed": 2.815417921686747, "loss_zero": 12.4375, "frac_recovered": 0.9485833960843374, "frac_alive": 0.98956298828125, "hyperparameters": {"n_inputs": 1000, "context_length": 1024}}
skip/ort_4/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4d10d00c08d3da33e1224386d9930182978aa1510f54f3e6798dae3283c2c6a3
3
+ size 1208232982
skip/ort_4/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "BatchTopKTrainer",
4
+ "dict_class": "BatchTopKSAE",
5
+ "lr": 0.0002,
6
+ "steps": 244140,
7
+ "auxk_alpha": 0.0625,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 195312,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 65536,
16
+ "k": 70,
17
+ "device": "cuda:1",
18
+ "layer": 12,
19
+ "lm_name": "google/gemma-2-2b",
20
+ "wandb_name": "BatchTopKOrtTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_0",
21
+ "submodule_name": "resid_post_layer_12"
22
+ },
23
+ "buffer": {
24
+ "d_submodule": 2304,
25
+ "io": "out",
26
+ "n_ctxs": 244,
27
+ "ctx_len": 1024,
28
+ "refresh_batch_size": 32,
29
+ "out_batch_size": 2048,
30
+ "device": "cuda:1"
31
+ }
32
+ }
skip/ort_4/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 51.19578313253012, "l1_loss": 506.03614457831327, "l0": 69.96392854437771, "frac_variance_explained": 0.8081466490963856, "cossim": 0.93359375, "l2_ratio": 0.93359375, "relative_reconstruction_bias": 1.0010824548192772, "loss_original": 2.2974868222891565, "loss_reconstructed": 2.8901543674698793, "loss_zero": 12.4375, "frac_recovered": 0.9414533132530121, "frac_alive": 0.990447998046875, "hyperparameters": {"n_inputs": 1000, "context_length": 1024}}
skip/ort_64/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:39c177f45d314453842a99f0d40ac8401d8448209858c5818855f55739a92caf
3
+ size 1208232982
skip/ort_64/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "BatchTopKTrainer",
4
+ "dict_class": "BatchTopKSAE",
5
+ "lr": 0.0002,
6
+ "steps": 244140,
7
+ "auxk_alpha": 0.0625,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 195312,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 65536,
16
+ "k": 70,
17
+ "device": "cuda:7",
18
+ "layer": 12,
19
+ "lm_name": "google/gemma-2-2b",
20
+ "wandb_name": "BatchTopKOrtTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_0",
21
+ "submodule_name": "resid_post_layer_12"
22
+ },
23
+ "buffer": {
24
+ "d_submodule": 2304,
25
+ "io": "out",
26
+ "n_ctxs": 244,
27
+ "ctx_len": 1024,
28
+ "refresh_batch_size": 32,
29
+ "out_batch_size": 2048,
30
+ "device": "cuda:7"
31
+ }
32
+ }
skip/ort_64/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 50.93975903614458, "l1_loss": 494.6144578313253, "l0": 69.70204498107175, "frac_variance_explained": 0.810570406626506, "cossim": 0.933617281626506, "l2_ratio": 0.93359375, "relative_reconstruction_bias": 1.000047063253012, "loss_original": 2.2974868222891565, "loss_reconstructed": 2.8665286144578315, "loss_zero": 12.4375, "frac_recovered": 0.9435005647590361, "frac_alive": 0.9888458251953125, "hyperparameters": {"n_inputs": 1000, "context_length": 1024}}
skip/ort_8/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a081bd4e8e79081f398d6576dc216af3630d9173110346475d90376dc92c6458
3
+ size 1208232982
skip/ort_8/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "BatchTopKTrainer",
4
+ "dict_class": "BatchTopKSAE",
5
+ "lr": 0.0002,
6
+ "steps": 244140,
7
+ "auxk_alpha": 0.0625,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 195312,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 65536,
16
+ "k": 70,
17
+ "device": "cuda:2",
18
+ "layer": 12,
19
+ "lm_name": "google/gemma-2-2b",
20
+ "wandb_name": "BatchTopKOrtTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_0",
21
+ "submodule_name": "resid_post_layer_12"
22
+ },
23
+ "buffer": {
24
+ "d_submodule": 2304,
25
+ "io": "out",
26
+ "n_ctxs": 244,
27
+ "ctx_len": 1024,
28
+ "refresh_batch_size": 32,
29
+ "out_batch_size": 2048,
30
+ "device": "cuda:2"
31
+ }
32
+ }
skip/ort_8/google_gemma-2-2b_batch_top_k_ort/resid_post_layer_12/trainer_0/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 51.20331325301205, "l1_loss": 504.71084337349396, "l0": 69.73826810538051, "frac_variance_explained": 0.8082407756024096, "cossim": 0.93359375, "l2_ratio": 0.93359375, "relative_reconstruction_bias": 1.0005176957831325, "loss_original": 2.2974868222891565, "loss_reconstructed": 2.9903990963855422, "loss_zero": 12.4375, "frac_recovered": 0.9311699924698795, "frac_alive": 0.9906005859375, "hyperparameters": {"n_inputs": 1000, "context_length": 1024}}