Add files using upload-large-folder tool

altmin_datasample_omp_k48_s1/results.json

@@ -0,0 +1,23 @@
+{
+  "init": "datasample",
+  "coding": "omp",
+  "k": 48,
+  "n_samples": 10000000,
+  "lr": 0.0003,
+  "seed": 1,
+  "eval": {
+    "true_l0": 34.465175,
+    "sae_l0": 33.416215,
+    "dead_latents": 0,
+    "shrinkage": 0.9272919580078125,
+    "explained_variance": 0.8575585358997201,
+    "mcc": 0.6327148675918579,
+    "uniqueness": 0.7420654296875,
+    "classification/precision": 0.688429057598114,
+    "classification/recall": 0.4184923470020294,
+    "classification/f1_score": 0.4094749093055725,
+    "classification/accuracy": 0.9838761687278748
+  },
+  "dec_fidelity_final": 0.6734853982925415,
+  "seconds": 3057.17805147171
+}

altmin_oracle_omp_k48_s1/results.json

@@ -0,0 +1,23 @@
+{
+  "init": "oracle",
+  "coding": "omp",
+  "k": 48,
+  "n_samples": 10000000,
+  "lr": 0.0003,
+  "seed": 1,
+  "eval": {
+    "true_l0": 34.461625,
+    "sae_l0": 38.44182,
+    "dead_latents": 0,
+    "shrinkage": 0.9756087316894532,
+    "explained_variance": 0.9379646703251204,
+    "mcc": 0.9614838361740112,
+    "uniqueness": 0.9976806640625,
+    "classification/precision": 0.7950587868690491,
+    "classification/recall": 0.7917520999908447,
+    "classification/f1_score": 0.7660229802131653,
+    "classification/accuracy": 0.9987500905990601
+  },
+  "dec_fidelity_final": 0.9632136821746826,
+  "seconds": 3055.872364282608
+}

l1/cfg.json

@@ -0,0 +1 @@
+{"d_in": 768, "apply_b_dec_to_input": true, "device": "cuda", "normalize_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "reshape_activations": "none", "d_sae": 16384, "dtype": "float32", "architecture": "standard"}

l1/ckpts/final_50003968/activation_scaler.json

@@ -0,0 +1 @@
+{"scaling_factor": null}

l1/ckpts/final_50003968/cfg.json

@@ -0,0 +1 @@
+{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "none", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "l1_coefficient": 5.0, "lp_norm": 1.0, "l1_warm_up_steps": 5000, "architecture": "standard"}

l1/ckpts/final_50003968/runner_config.json

@@ -0,0 +1,58 @@
+{
+  "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1",
+  "sae": {
+    "d_in": 768,
+    "d_sae": 16384,
+    "dtype": "float32",
+    "device": "cuda",
+    "apply_b_dec_to_input": true,
+    "normalize_activations": "none",
+    "reshape_activations": "none",
+    "metadata": {
+      "sae_lens_version": "6.43.0",
+      "sae_lens_training_version": "6.43.0"
+    },
+    "decoder_init_norm": 0.1,
+    "l1_coefficient": 5.0,
+    "lp_norm": 1.0,
+    "l1_warm_up_steps": 5000,
+    "architecture": "standard"
+  },
+  "training_samples": 50000000,
+  "batch_size": 4096,
+  "lr": 0.0003,
+  "lr_warm_up_steps": 1000,
+  "lr_decay_steps": 0,
+  "lr_scheduler_name": "constant",
+  "lr_end": 2.9999999999999997e-05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "n_restart_cycles": 1,
+  "device": "cuda",
+  "autocast_sae": false,
+  "autocast_data": false,
+  "n_checkpoints": 0,
+  "checkpoint_path": "runs/l1/ckpts",
+  "save_final_checkpoint": true,
+  "output_path": "runs/l1",
+  "save_synthetic_model": false,
+  "eval_frequency": 200000,
+  "eval_samples": 200000,
+  "run_final_eval": true,
+  "dead_feature_window": 1000,
+  "feature_sampling_window": 2000,
+  "n_batches_for_norm_estimate": 1000,
+  "sae_lens_version": "6.43.0",
+  "logger": {
+    "log_to_wandb": true,
+    "log_activations_store_to_wandb": false,
+    "log_optimizer_state_to_wandb": false,
+    "log_weights_to_wandb": true,
+    "wandb_project": "sae_lens_training",
+    "wandb_id": null,
+    "run_name": "synthetic-standard-16384-LR-0.0003",
+    "wandb_entity": null,
+    "wandb_log_frequency": 10,
+    "eval_every_n_wandb_logs": 100
+  }
+}

l1/eval_stats.json

@@ -0,0 +1,15 @@
+{
+  "true_l0": 34.472315,
+  "sae_l0": 28.09419,
+  "dead_latents": 2586,
+  "shrinkage": 0.6749181384277344,
+  "explained_variance": 0.7596865098641346,
+  "mcc": 0.3895021378993988,
+  "uniqueness": 0.4180908203125,
+  "classification": {
+    "precision": 0.5969710350036621,
+    "recall": 0.4199705123901367,
+    "f1_score": 0.38069725036621094,
+    "accuracy": 0.9573115706443787
+  }
+}

l1/meta.json

@@ -0,0 +1,6 @@
+{
+  "variant": "l1",
+  "n_samples": 50000000,
+  "seconds": 1246.3810708522797,
+  "out": "runs/l1"
+}

l1/metrics.json

@@ -0,0 +1,20 @@
+{
+  "geometric": {
+    "max_cos_mean": 0.4059242606163025,
+    "max_cos_median": 0.49422556161880493,
+    "frac_recovered_0.9": 0.02142333984375,
+    "frac_recovered_0.95": 0.0205078125,
+    "hungarian_mean": 0.387668251991272,
+    "hungarian_frac_0.9": 0.02142333984375
+  },
+  "sae_lens_eval": {
+    "true_l0": 34.459226,
+    "sae_l0": 28.09275,
+    "dead_latents": 2179,
+    "shrinkage": 0.6752423949584961,
+    "explained_variance": 0.7597061484071307,
+    "mcc": 0.3895021378993988,
+    "uniqueness": 0.4180908203125,
+    "classification": "<class 'dict'>"
+  }
+}

l1/runner_config.json

@@ -0,0 +1,58 @@
+{
+  "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1",
+  "sae": {
+    "d_in": 768,
+    "d_sae": 16384,
+    "dtype": "float32",
+    "device": "cuda",
+    "apply_b_dec_to_input": true,
+    "normalize_activations": "none",
+    "reshape_activations": "none",
+    "metadata": {
+      "sae_lens_version": "6.43.0",
+      "sae_lens_training_version": "6.43.0"
+    },
+    "decoder_init_norm": 0.1,
+    "l1_coefficient": 5.0,
+    "lp_norm": 1.0,
+    "l1_warm_up_steps": 5000,
+    "architecture": "standard"
+  },
+  "training_samples": 50000000,
+  "batch_size": 4096,
+  "lr": 0.0003,
+  "lr_warm_up_steps": 1000,
+  "lr_decay_steps": 0,
+  "lr_scheduler_name": "constant",
+  "lr_end": 2.9999999999999997e-05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "n_restart_cycles": 1,
+  "device": "cuda",
+  "autocast_sae": false,
+  "autocast_data": false,
+  "n_checkpoints": 0,
+  "checkpoint_path": "runs/l1/ckpts",
+  "save_final_checkpoint": true,
+  "output_path": "runs/l1",
+  "save_synthetic_model": false,
+  "eval_frequency": 200000,
+  "eval_samples": 200000,
+  "run_final_eval": true,
+  "dead_feature_window": 1000,
+  "feature_sampling_window": 2000,
+  "n_batches_for_norm_estimate": 1000,
+  "sae_lens_version": "6.43.0",
+  "logger": {
+    "log_to_wandb": true,
+    "log_activations_store_to_wandb": false,
+    "log_optimizer_state_to_wandb": false,
+    "log_weights_to_wandb": true,
+    "wandb_project": "sae_lens_training",
+    "wandb_id": null,
+    "run_name": "synthetic-standard-16384-LR-0.0003",
+    "wandb_entity": null,
+    "wandb_log_frequency": 10,
+    "eval_every_n_wandb_logs": 100
+  }
+}

l1_l1c10.0_s0/cfg.json

@@ -0,0 +1 @@
+{"dtype": "float32", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "reshape_activations": "none", "apply_b_dec_to_input": true, "device": "cuda", "normalize_activations": "none", "d_sae": 16384, "d_in": 768, "architecture": "standard"}

l1_l1c10.0_s0/ckpts/final_50003968/activation_scaler.json

@@ -0,0 +1 @@
+{"scaling_factor": null}

l1_l1c10.0_s0/ckpts/final_50003968/cfg.json

@@ -0,0 +1 @@
+{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "none", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "l1_coefficient": 10.0, "lp_norm": 1.0, "l1_warm_up_steps": 5000, "architecture": "standard"}

l1_l1c10.0_s0/ckpts/final_50003968/runner_config.json

@@ -0,0 +1,58 @@
+{
+  "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1",
+  "sae": {
+    "d_in": 768,
+    "d_sae": 16384,
+    "dtype": "float32",
+    "device": "cuda",
+    "apply_b_dec_to_input": true,
+    "normalize_activations": "none",
+    "reshape_activations": "none",
+    "metadata": {
+      "sae_lens_version": "6.43.0",
+      "sae_lens_training_version": "6.43.0"
+    },
+    "decoder_init_norm": 0.1,
+    "l1_coefficient": 10.0,
+    "lp_norm": 1.0,
+    "l1_warm_up_steps": 5000,
+    "architecture": "standard"
+  },
+  "training_samples": 50000000,
+  "batch_size": 4096,
+  "lr": 0.0003,
+  "lr_warm_up_steps": 1000,
+  "lr_decay_steps": 0,
+  "lr_scheduler_name": "constant",
+  "lr_end": 2.9999999999999997e-05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "n_restart_cycles": 1,
+  "device": "cuda",
+  "autocast_sae": false,
+  "autocast_data": false,
+  "n_checkpoints": 0,
+  "checkpoint_path": "runs/l1_l1c10.0_s0/ckpts",
+  "save_final_checkpoint": true,
+  "output_path": "runs/l1_l1c10.0_s0",
+  "save_synthetic_model": false,
+  "eval_frequency": 200000,
+  "eval_samples": 200000,
+  "run_final_eval": true,
+  "dead_feature_window": 1000,
+  "feature_sampling_window": 2000,
+  "n_batches_for_norm_estimate": 1000,
+  "sae_lens_version": "6.43.0",
+  "logger": {
+    "log_to_wandb": true,
+    "log_activations_store_to_wandb": false,
+    "log_optimizer_state_to_wandb": false,
+    "log_weights_to_wandb": true,
+    "wandb_project": "sae_lens_training",
+    "wandb_id": null,
+    "run_name": "synthetic-standard-16384-LR-0.0003",
+    "wandb_entity": null,
+    "wandb_log_frequency": 10,
+    "eval_every_n_wandb_logs": 100
+  }
+}

l1_l1c10.0_s0/eval_stats.json

@@ -0,0 +1,15 @@
+{
+  "true_l0": 34.466205,
+  "sae_l0": 14.88724,
+  "dead_latents": 5648,
+  "shrinkage": 0.4484661962890625,
+  "explained_variance": 0.5605491484812328,
+  "mcc": 0.3398200273513794,
+  "uniqueness": 0.30902099609375,
+  "classification": {
+    "precision": 0.542731761932373,
+    "recall": 0.1990332156419754,
+    "f1_score": 0.22677792608737946,
+    "accuracy": 0.9533296227455139
+  }
+}

l1_l1c10.0_s0/meta.json

@@ -0,0 +1,9 @@
+{
+  "variant": "l1",
+  "k": 32,
+  "l1": 10.0,
+  "seed": 0,
+  "n_samples": 50000000,
+  "seconds": 1275.7077553272247,
+  "out": "runs/l1_l1c10.0_s0"
+}

l1_l1c10.0_s0/runner_config.json

@@ -0,0 +1,58 @@
+{
+  "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1",
+  "sae": {
+    "d_in": 768,
+    "d_sae": 16384,
+    "dtype": "float32",
+    "device": "cuda",
+    "apply_b_dec_to_input": true,
+    "normalize_activations": "none",
+    "reshape_activations": "none",
+    "metadata": {
+      "sae_lens_version": "6.43.0",
+      "sae_lens_training_version": "6.43.0"
+    },
+    "decoder_init_norm": 0.1,
+    "l1_coefficient": 10.0,
+    "lp_norm": 1.0,
+    "l1_warm_up_steps": 5000,
+    "architecture": "standard"
+  },
+  "training_samples": 50000000,
+  "batch_size": 4096,
+  "lr": 0.0003,
+  "lr_warm_up_steps": 1000,
+  "lr_decay_steps": 0,
+  "lr_scheduler_name": "constant",
+  "lr_end": 2.9999999999999997e-05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "n_restart_cycles": 1,
+  "device": "cuda",
+  "autocast_sae": false,
+  "autocast_data": false,
+  "n_checkpoints": 0,
+  "checkpoint_path": "runs/l1_l1c10.0_s0/ckpts",
+  "save_final_checkpoint": true,
+  "output_path": "runs/l1_l1c10.0_s0",
+  "save_synthetic_model": false,
+  "eval_frequency": 200000,
+  "eval_samples": 200000,
+  "run_final_eval": true,
+  "dead_feature_window": 1000,
+  "feature_sampling_window": 2000,
+  "n_batches_for_norm_estimate": 1000,
+  "sae_lens_version": "6.43.0",
+  "logger": {
+    "log_to_wandb": true,
+    "log_activations_store_to_wandb": false,
+    "log_optimizer_state_to_wandb": false,
+    "log_weights_to_wandb": true,
+    "wandb_project": "sae_lens_training",
+    "wandb_id": null,
+    "run_name": "synthetic-standard-16384-LR-0.0003",
+    "wandb_entity": null,
+    "wandb_log_frequency": 10,
+    "eval_every_n_wandb_logs": 100
+  }
+}

p2_joint_k48_s1/results.json

@@ -0,0 +1,19 @@
+{
+  "variant": "joint",
+  "k": 48,
+  "n_samples": 30000000,
+  "warmup_samples": 0,
+  "lr_enc": 0.0003,
+  "lr_dec": 3e-05,
+  "seed": 1,
+  "eval": {
+    "true_l0": 34.48436,
+    "sae_l0": 48.0,
+    "dead_latents": 3153,
+    "shrinkage": 0.9297531726074219,
+    "explained_variance": 0.8629156510529934,
+    "mcc": 0.466611385345459,
+    "uniqueness": 0.68023681640625
+  },
+  "seconds": 525.7930476665497
+}

topk_k40_s1/ckpts/final_50003968/activation_scaler.json

@@ -0,0 +1 @@
+{"scaling_factor": null}

topk_k40_s1/ckpts/final_50003968/runner_config.json

@@ -0,0 +1,59 @@
+{
+  "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1",
+  "sae": {
+    "d_in": 768,
+    "d_sae": 16384,
+    "dtype": "float32",
+    "device": "cuda",
+    "apply_b_dec_to_input": true,
+    "normalize_activations": "none",
+    "reshape_activations": "none",
+    "metadata": {
+      "sae_lens_version": "6.43.0",
+      "sae_lens_training_version": "6.43.0"
+    },
+    "decoder_init_norm": 0.1,
+    "k": 40,
+    "use_sparse_activations": false,
+    "aux_loss_coefficient": 1.0,
+    "rescale_acts_by_decoder_norm": true,
+    "architecture": "topk"
+  },
+  "training_samples": 50000000,
+  "batch_size": 4096,
+  "lr": 0.0003,
+  "lr_warm_up_steps": 1000,
+  "lr_decay_steps": 0,
+  "lr_scheduler_name": "constant",
+  "lr_end": 2.9999999999999997e-05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "n_restart_cycles": 1,
+  "device": "cuda",
+  "autocast_sae": false,
+  "autocast_data": false,
+  "n_checkpoints": 0,
+  "checkpoint_path": "runs/topk_k40_s1/ckpts",
+  "save_final_checkpoint": true,
+  "output_path": "runs/topk_k40_s1",
+  "save_synthetic_model": false,
+  "eval_frequency": 200000,
+  "eval_samples": 200000,
+  "run_final_eval": true,
+  "dead_feature_window": 1000,
+  "feature_sampling_window": 2000,
+  "n_batches_for_norm_estimate": 1000,
+  "sae_lens_version": "6.43.0",
+  "logger": {
+    "log_to_wandb": true,
+    "log_activations_store_to_wandb": false,
+    "log_optimizer_state_to_wandb": false,
+    "log_weights_to_wandb": true,
+    "wandb_project": "sae_lens_training",
+    "wandb_id": null,
+    "run_name": "synthetic-topk-16384-LR-0.0003",
+    "wandb_entity": null,
+    "wandb_log_frequency": 10,
+    "eval_every_n_wandb_logs": 100
+  }
+}

topk_k48_s1/barrier.json

@@ -0,0 +1,52 @@
+{
+  "run": "runs/topk_k48_s1",
+  "k": 48,
+  "n_samples": 5000,
+  "aligned_cos_mean": 0.6954070329666138,
+  "alphas": [
+    0.0,
+    0.05,
+    0.1,
+    0.15000000000000002,
+    0.2,
+    0.25,
+    0.30000000000000004,
+    0.35000000000000003,
+    0.4,
+    0.45,
+    0.5,
+    0.55,
+    0.6000000000000001,
+    0.65,
+    0.7000000000000001,
+    0.75,
+    0.8,
+    0.8500000000000001,
+    0.9,
+    0.9500000000000001,
+    1.0
+  ],
+  "mse": [
+    0.02737385779619217,
+    0.027973411604762077,
+    0.030088335275650024,
+    0.033576928079128265,
+    0.038279127329587936,
+    0.043935250490903854,
+    0.0500849112868309,
+    0.05611300840973854,
+    0.06107799708843231,
+    0.06396925449371338,
+    0.06423264741897583,
+    0.06207716837525368,
+    0.057816680520772934,
+    0.05181102827191353,
+    0.04433048516511917,
+    0.03647156059741974,
+    0.029172265902161598,
+    0.0230012908577919,
+    0.018400993198156357,
+    0.01567491888999939,
+    0.014970618300139904
+  ]
+}

topk_k48_s1/cfg.json

@@ -0,0 +1 @@
+{"reshape_activations": "none", "d_in": 768, "normalize_activations": "none", "d_sae": 16384, "k": 48, "apply_b_dec_to_input": true, "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "dtype": "float32", "rescale_acts_by_decoder_norm": true, "device": "cuda", "architecture": "topk"}

topk_k48_s1/ckpts/final_50003968/activation_scaler.json

@@ -0,0 +1 @@
+{"scaling_factor": null}

topk_k48_s1/ckpts/final_50003968/cfg.json

@@ -0,0 +1 @@
+{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "none", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 48, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"}

topk_k48_s1/ckpts/final_50003968/runner_config.json

@@ -0,0 +1,59 @@
+{
+  "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1",
+  "sae": {
+    "d_in": 768,
+    "d_sae": 16384,
+    "dtype": "float32",
+    "device": "cuda",
+    "apply_b_dec_to_input": true,
+    "normalize_activations": "none",
+    "reshape_activations": "none",
+    "metadata": {
+      "sae_lens_version": "6.43.0",
+      "sae_lens_training_version": "6.43.0"
+    },
+    "decoder_init_norm": 0.1,
+    "k": 48,
+    "use_sparse_activations": false,
+    "aux_loss_coefficient": 1.0,
+    "rescale_acts_by_decoder_norm": true,
+    "architecture": "topk"
+  },
+  "training_samples": 50000000,
+  "batch_size": 4096,
+  "lr": 0.0003,
+  "lr_warm_up_steps": 1000,
+  "lr_decay_steps": 0,
+  "lr_scheduler_name": "constant",
+  "lr_end": 2.9999999999999997e-05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "n_restart_cycles": 1,
+  "device": "cuda",
+  "autocast_sae": false,
+  "autocast_data": false,
+  "n_checkpoints": 0,
+  "checkpoint_path": "runs/topk_k48_s1/ckpts",
+  "save_final_checkpoint": true,
+  "output_path": "runs/topk_k48_s1",
+  "save_synthetic_model": false,
+  "eval_frequency": 200000,
+  "eval_samples": 200000,
+  "run_final_eval": true,
+  "dead_feature_window": 1000,
+  "feature_sampling_window": 2000,
+  "n_batches_for_norm_estimate": 1000,
+  "sae_lens_version": "6.43.0",
+  "logger": {
+    "log_to_wandb": true,
+    "log_activations_store_to_wandb": false,
+    "log_optimizer_state_to_wandb": false,
+    "log_weights_to_wandb": true,
+    "wandb_project": "sae_lens_training",
+    "wandb_id": null,
+    "run_name": "synthetic-topk-16384-LR-0.0003",
+    "wandb_entity": null,
+    "wandb_log_frequency": 10,
+    "eval_every_n_wandb_logs": 100
+  }
+}

topk_k48_s1/eval_stats.json

@@ -0,0 +1,15 @@
+{
+  "true_l0": 34.512515,
+  "sae_l0": 48.0,
+  "dead_latents": 2308,
+  "shrinkage": 0.9608987731933594,
+  "explained_variance": 0.9261054240759901,
+  "mcc": 0.6978566646575928,
+  "uniqueness": 0.801025390625,
+  "classification": {
+    "precision": 0.4849301278591156,
+    "recall": 0.7459948658943176,
+    "f1_score": 0.5721877217292786,
+    "accuracy": 0.9968123435974121
+  }
+}

topk_k48_s1/meta.json

@@ -0,0 +1,9 @@
+{
+  "variant": "topk",
+  "k": 48,
+  "l1": 5.0,
+  "seed": 1,
+  "n_samples": 50000000,
+  "seconds": 1433.1707093715668,
+  "out": "runs/topk_k48_s1"
+}

topk_k48_s1/metrics.json

@@ -0,0 +1,20 @@
+{
+  "geometric": {
+    "max_cos_mean": 0.7426820993423462,
+    "max_cos_median": 0.8097807765007019,
+    "frac_recovered_0.9": 0.32293701171875,
+    "frac_recovered_0.95": 0.2510986328125,
+    "hungarian_mean": 0.6954070329666138,
+    "hungarian_frac_0.9": 0.3228759765625
+  },
+  "sae_lens_eval": {
+    "true_l0": 34.482116,
+    "sae_l0": 48.0,
+    "dead_latents": 1910,
+    "shrinkage": 0.9608231494140625,
+    "explained_variance": 0.9261436378060115,
+    "mcc": 0.6978566646575928,
+    "uniqueness": 0.801025390625,
+    "classification": "<class 'dict'>"
+  }
+}

topk_k48_s1/mode_connectivity.json

@@ -0,0 +1,57 @@
+{
+  "run": "runs/topk_k48_s1",
+  "k": 48,
+  "train_alphas": [
+    0.2,
+    0.4,
+    0.5,
+    0.6,
+    0.8
+  ],
+  "alphas": [
+    0.0,
+    0.05,
+    0.1,
+    0.15000000000000002,
+    0.2,
+    0.25,
+    0.30000000000000004,
+    0.35000000000000003,
+    0.4,
+    0.45,
+    0.5,
+    0.55,
+    0.6000000000000001,
+    0.65,
+    0.7000000000000001,
+    0.75,
+    0.8,
+    0.8500000000000001,
+    0.9,
+    0.9500000000000001,
+    1.0
+  ],
+  "optimised_mse": [
+    0.0278778076171875,
+    0.028087442740797997,
+    0.029514452442526817,
+    0.03136105090379715,
+    0.033336397260427475,
+    0.03843143954873085,
+    0.038147639483213425,
+    0.03801201656460762,
+    0.038283515721559525,
+    0.038278572261333466,
+    0.037768322974443436,
+    0.03660793602466583,
+    0.034817613661289215,
+    0.03231758251786232,
+    0.029309000819921494,
+    0.02605382911860943,
+    0.022752316668629646,
+    0.019668152555823326,
+    0.016921646893024445,
+    0.014870095066726208,
+    0.014269914478063583
+  ]
+}

topk_k48_s1/runner_config.json

@@ -0,0 +1,59 @@
+{
+  "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1",
+  "sae": {
+    "d_in": 768,
+    "d_sae": 16384,
+    "dtype": "float32",
+    "device": "cuda",
+    "apply_b_dec_to_input": true,
+    "normalize_activations": "none",
+    "reshape_activations": "none",
+    "metadata": {
+      "sae_lens_version": "6.43.0",
+      "sae_lens_training_version": "6.43.0"
+    },
+    "decoder_init_norm": 0.1,
+    "k": 48,
+    "use_sparse_activations": false,
+    "aux_loss_coefficient": 1.0,
+    "rescale_acts_by_decoder_norm": true,
+    "architecture": "topk"
+  },
+  "training_samples": 50000000,
+  "batch_size": 4096,
+  "lr": 0.0003,
+  "lr_warm_up_steps": 1000,
+  "lr_decay_steps": 0,
+  "lr_scheduler_name": "constant",
+  "lr_end": 2.9999999999999997e-05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "n_restart_cycles": 1,
+  "device": "cuda",
+  "autocast_sae": false,
+  "autocast_data": false,
+  "n_checkpoints": 0,
+  "checkpoint_path": "runs/topk_k48_s1/ckpts",
+  "save_final_checkpoint": true,
+  "output_path": "runs/topk_k48_s1",
+  "save_synthetic_model": false,
+  "eval_frequency": 200000,
+  "eval_samples": 200000,
+  "run_final_eval": true,
+  "dead_feature_window": 1000,
+  "feature_sampling_window": 2000,
+  "n_batches_for_norm_estimate": 1000,
+  "sae_lens_version": "6.43.0",
+  "logger": {
+    "log_to_wandb": true,
+    "log_activations_store_to_wandb": false,
+    "log_optimizer_state_to_wandb": false,
+    "log_weights_to_wandb": true,
+    "wandb_project": "sae_lens_training",
+    "wandb_id": null,
+    "run_name": "synthetic-topk-16384-LR-0.0003",
+    "wandb_entity": null,
+    "wandb_log_frequency": 10,
+    "eval_every_n_wandb_logs": 100
+  }
+}

topk_k48_s2/barrier.json

@@ -0,0 +1,52 @@
+{
+  "run": "runs/topk_k48_s2",
+  "k": 48,
+  "n_samples": 5000,
+  "aligned_cos_mean": 0.6940204501152039,
+  "alphas": [
+    0.0,
+    0.05,
+    0.1,
+    0.15000000000000002,
+    0.2,
+    0.25,
+    0.30000000000000004,
+    0.35000000000000003,
+    0.4,
+    0.45,
+    0.5,
+    0.55,
+    0.6000000000000001,
+    0.65,
+    0.7000000000000001,
+    0.75,
+    0.8,
+    0.8500000000000001,
+    0.9,
+    0.9500000000000001,
+    1.0
+  ],
+  "mse": [
+    0.02816515415906906,
+    0.02877231128513813,
+    0.030867310240864754,
+    0.03433920070528984,
+    0.03899581357836723,
+    0.04459444805979729,
+    0.05060192570090294,
+    0.056368596851825714,
+    0.06094635650515556,
+    0.06333129107952118,
+    0.06322220712900162,
+    0.06079515069723129,
+    0.05642535910010338,
+    0.05043580383062363,
+    0.043033063411712646,
+    0.03533760458230972,
+    0.028203310444951057,
+    0.022138960659503937,
+    0.017610330134630203,
+    0.014930839650332928,
+    0.0142666669562459
+  ]
+}

topk_k48_s2/cfg.json

@@ -0,0 +1 @@
+{"device": "cuda", "rescale_acts_by_decoder_norm": true, "k": 48, "normalize_activations": "none", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "dtype": "float32", "apply_b_dec_to_input": true, "d_in": 768, "d_sae": 16384, "architecture": "topk"}

topk_k48_s2/ckpts/final_50003968/activation_scaler.json

@@ -0,0 +1 @@
+{"scaling_factor": null}

topk_k48_s2/ckpts/final_50003968/cfg.json

@@ -0,0 +1 @@
+{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "none", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 48, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"}

topk_k48_s2/ckpts/final_50003968/runner_config.json

@@ -0,0 +1,59 @@
+{
+  "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1",
+  "sae": {
+    "d_in": 768,
+    "d_sae": 16384,
+    "dtype": "float32",
+    "device": "cuda",
+    "apply_b_dec_to_input": true,
+    "normalize_activations": "none",
+    "reshape_activations": "none",
+    "metadata": {
+      "sae_lens_version": "6.43.0",
+      "sae_lens_training_version": "6.43.0"
+    },
+    "decoder_init_norm": 0.1,
+    "k": 48,
+    "use_sparse_activations": false,
+    "aux_loss_coefficient": 1.0,
+    "rescale_acts_by_decoder_norm": true,
+    "architecture": "topk"
+  },
+  "training_samples": 50000000,
+  "batch_size": 4096,
+  "lr": 0.0003,
+  "lr_warm_up_steps": 1000,
+  "lr_decay_steps": 0,
+  "lr_scheduler_name": "constant",
+  "lr_end": 2.9999999999999997e-05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "n_restart_cycles": 1,
+  "device": "cuda",
+  "autocast_sae": false,
+  "autocast_data": false,
+  "n_checkpoints": 0,
+  "checkpoint_path": "runs/topk_k48_s2/ckpts",
+  "save_final_checkpoint": true,
+  "output_path": "runs/topk_k48_s2",
+  "save_synthetic_model": false,
+  "eval_frequency": 200000,
+  "eval_samples": 200000,
+  "run_final_eval": true,
+  "dead_feature_window": 1000,
+  "feature_sampling_window": 2000,
+  "n_batches_for_norm_estimate": 1000,
+  "sae_lens_version": "6.43.0",
+  "logger": {
+    "log_to_wandb": true,
+    "log_activations_store_to_wandb": false,
+    "log_optimizer_state_to_wandb": false,
+    "log_weights_to_wandb": true,
+    "wandb_project": "sae_lens_training",
+    "wandb_id": null,
+    "run_name": "synthetic-topk-16384-LR-0.0003",
+    "wandb_entity": null,
+    "wandb_log_frequency": 10,
+    "eval_every_n_wandb_logs": 100
+  }
+}

topk_k48_s2/eval_stats.json

@@ -0,0 +1,15 @@
+{
+  "true_l0": 34.46119,
+  "sae_l0": 48.0,
+  "dead_latents": 2313,
+  "shrinkage": 0.9596230041503906,
+  "explained_variance": 0.9260622354368173,
+  "mcc": 0.6963987350463867,
+  "uniqueness": 0.804443359375,
+  "classification": {
+    "precision": 0.48351460695266724,
+    "recall": 0.7443896532058716,
+    "f1_score": 0.5715254545211792,
+    "accuracy": 0.9969232082366943
+  }
+}

topk_k48_s2/meta.json

@@ -0,0 +1,9 @@
+{
+  "variant": "topk",
+  "k": 48,
+  "l1": 5.0,
+  "seed": 2,
+  "n_samples": 50000000,
+  "seconds": 1451.632717370987,
+  "out": "runs/topk_k48_s2"
+}

topk_k48_s2/metrics.json

@@ -0,0 +1,20 @@
+{
+  "geometric": {
+    "max_cos_mean": 0.7417675256729126,
+    "max_cos_median": 0.8087477087974548,
+    "frac_recovered_0.9": 0.31982421875,
+    "frac_recovered_0.95": 0.249267578125,
+    "hungarian_mean": 0.6940203905105591,
+    "hungarian_frac_0.9": 0.31976318359375
+  },
+  "sae_lens_eval": {
+    "true_l0": 34.462924,
+    "sae_l0": 48.0,
+    "dead_latents": 1904,
+    "shrinkage": 0.9596458997802735,
+    "explained_variance": 0.9262095513537529,
+    "mcc": 0.6963987350463867,
+    "uniqueness": 0.804443359375,
+    "classification": "<class 'dict'>"
+  }
+}

topk_k48_s2/mode_connectivity.json

@@ -0,0 +1,57 @@
+{
+  "run": "runs/topk_k48_s2",
+  "k": 48,
+  "train_alphas": [
+    0.2,
+    0.4,
+    0.5,
+    0.6,
+    0.8
+  ],
+  "alphas": [
+    0.0,
+    0.05,
+    0.1,
+    0.15000000000000002,
+    0.2,
+    0.25,
+    0.30000000000000004,
+    0.35000000000000003,
+    0.4,
+    0.45,
+    0.5,
+    0.55,
+    0.6000000000000001,
+    0.65,
+    0.7000000000000001,
+    0.75,
+    0.8,
+    0.8500000000000001,
+    0.9,
+    0.9500000000000001,
+    1.0
+  ],
+  "optimised_mse": [
+    0.027076900005340576,
+    0.027401672676205635,
+    0.029085956513881683,
+    0.03135727345943451,
+    0.033744651824235916,
+    0.0359901487827301,
+    0.03787828981876373,
+    0.039249420166015625,
+    0.03998696431517601,
+    0.03997647762298584,
+    0.039204761385917664,
+    0.037724703550338745,
+    0.035543449223041534,
+    0.03279434144496918,
+    0.02966124750673771,
+    0.026313742622733116,
+    0.022886032238602638,
+    0.01957164891064167,
+    0.016496820375323296,
+    0.01408455241471529,
+    0.013375862501561642
+  ]
+}

topk_k48_s2/runner_config.json

@@ -0,0 +1,59 @@
+{
+  "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1",
+  "sae": {
+    "d_in": 768,
+    "d_sae": 16384,
+    "dtype": "float32",
+    "device": "cuda",
+    "apply_b_dec_to_input": true,
+    "normalize_activations": "none",
+    "reshape_activations": "none",
+    "metadata": {
+      "sae_lens_version": "6.43.0",
+      "sae_lens_training_version": "6.43.0"
+    },
+    "decoder_init_norm": 0.1,
+    "k": 48,
+    "use_sparse_activations": false,
+    "aux_loss_coefficient": 1.0,
+    "rescale_acts_by_decoder_norm": true,
+    "architecture": "topk"
+  },
+  "training_samples": 50000000,
+  "batch_size": 4096,
+  "lr": 0.0003,
+  "lr_warm_up_steps": 1000,
+  "lr_decay_steps": 0,
+  "lr_scheduler_name": "constant",
+  "lr_end": 2.9999999999999997e-05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "n_restart_cycles": 1,
+  "device": "cuda",
+  "autocast_sae": false,
+  "autocast_data": false,
+  "n_checkpoints": 0,
+  "checkpoint_path": "runs/topk_k48_s2/ckpts",
+  "save_final_checkpoint": true,
+  "output_path": "runs/topk_k48_s2",
+  "save_synthetic_model": false,
+  "eval_frequency": 200000,
+  "eval_samples": 200000,
+  "run_final_eval": true,
+  "dead_feature_window": 1000,
+  "feature_sampling_window": 2000,
+  "n_batches_for_norm_estimate": 1000,
+  "sae_lens_version": "6.43.0",
+  "logger": {
+    "log_to_wandb": true,
+    "log_activations_store_to_wandb": false,
+    "log_optimizer_state_to_wandb": false,
+    "log_weights_to_wandb": true,
+    "wandb_project": "sae_lens_training",
+    "wandb_id": null,
+    "run_name": "synthetic-topk-16384-LR-0.0003",
+    "wandb_entity": null,
+    "wandb_log_frequency": 10,
+    "eval_every_n_wandb_logs": 100
+  }
+}