Release crosscoder + SAE + attribution artifacts

README.md

@@ -0,0 +1,83 @@
+---
+license: apache-2.0
+tags:
+- crosscoder
+- sparse-autoencoder
+- mech-interp
+- parameter-trajectory
+- pythia
+- olmo
+---
+
+# Parameter-trajectory crosscoders for vocabulary readout evolution
+
+Companion checkpoints to the paper *Parameter-trajectory crosscoders for
+vocabulary readout evolution* (NeurIPS 2026; arXiv:XXXX.XXXXX).
+
+We train **snapshot crosscoders** on parameter tensors (rather than
+activations) sampled across pretraining checkpoints. In Pythia-160M's output
+unembedding $W_U$, this reveals a sparse, reproducible **readout
+consolidation event** at training step ~1,000.
+
+## Repository layout
+
+| What you want | Path |
+|---|---|
+| Headline 5-seed Pythia-160M $W_U$ crosscoder | `pythia-160m/W_U/cross-snapshot-32/d8192/seed{0..4}.safetensors` |
+| 32 single-snapshot SAEs (in-time view) | `pythia-160m/W_U/per-snapshot-saes/d8192/step*.safetensors` |
+| Higher-capacity Pythia-160M $W_U$ crosscoders | `pythia-160m/W_U/cross-snapshot-32/d{16384,24576}/` |
+| Architecture-invariance comparison (vs. JumpReLU baseline) | `pythia-160m/W_U/architecture-comparison/d8192/{batchtopk,gated,gated-retuned}/` |
+| Snapshot-density downsample | `pythia-160m/W_U/cross-snapshot-16/d8192/` |
+| Final-snapshot-only SAE Pareto | `pythia-160m/W_U/final-snapshot-saes/d{6144..65536}.safetensors` |
+| Read/write asymmetry: $W_E$ side | `pythia-160m/W_E/cross-snapshot-32/d{8192,24576}/` |
+| Cross-scale (Pythia-1B) | `pythia-1b/W_U/cross-snapshot-32/d{8192,16384,24576}/` |
+| Larger-scale (Pythia-6.9B) | `pythia-6.9b/W_U/cross-snapshot-32/d32768/` |
+| Cross-family (OLMo-2-7B) | `olmo-2-7b/W_U/cross-snapshot-32/d32768/` |
+| §5 attribution-patching artifacts | `attribution/pythia-160m/` |
+
+Each directory contains `<name>.safetensors` (weights),
+`<name>.config.json` (training hparams + quality metrics), and `<name>.md`
+(one-page model card). The metadata format is unified across all artifacts.
+
+## Quick start
+
+```python
+from huggingface_hub import hf_hub_download
+from safetensors.torch import load_file
+import json
+
+# Download the headline Pythia-160M crosscoder, seed 0
+weights_path = hf_hub_download(
+    "matteohe/parameter-trajectory-crosscoders",
+    "pythia-160m/W_U/cross-snapshot-32/d8192/seed0.safetensors",
+)
+config_path = hf_hub_download(
+    "matteohe/parameter-trajectory-crosscoders",
+    "pythia-160m/W_U/cross-snapshot-32/d8192/seed0.config.json",
+)
+weights = load_file(weights_path)  # dict[str, Tensor]
+config = json.load(open(config_path))
+print(config["public_label"])  # "Pythia-160M W_U crosscoder, ..."
+```
+
+The crosscoder weight tensors are:
+- `W_E`: encoder, shape $(K, d_{\text{model}}, d_{\text{sae}})$
+- `W_D`: decoder, shape $(K, d_{\text{sae}}, d_{\text{model}})$
+- `b_E`: encoder bias, shape $(K, d_{\text{sae}})$
+- `b_D`: decoder bias, shape $(K, d_{\text{model}})$
+- `activation_function.log_jumprelu_threshold`: shape $(d_{\text{sae}},)$
+
+For per-snapshot SAEs `K=1`. The companion paper code (open-source, see
+`https://github.com/...`) provides `wu_adapter.build_crosscoder` which
+reconstructs the full module.
+
+## Citation
+
+```bibtex
+@inproceedings{{...,
+  title={{Parameter-trajectory crosscoders for vocabulary readout evolution}},
+  author={{...}},
+  booktitle={{NeurIPS 2026}},
+  year={{2026}},
+}}
+```

attribution/pythia-160m/induction-results.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88048a0c9972a962695610c19820d716b541807022c78a0f8a86557bc15346a1
+size 32822581

attribution/pythia-160m/induction-verdict.json

@@ -0,0 +1,11 @@
+{
+  "passed": true,
+  "max_ratio": 696.25,
+  "ratio_at_k": 32,
+  "ig_clean": true,
+  "n_samples": 1000,
+  "n_ig_steps": 10,
+  "snapshot_step": 143000,
+  "checkpoint": "/home/mh2274/rds/hpc-work/wu_crosscoder/run3_ge_exact_results/wu_cc_ge_exact_seed0.pt",
+  "wallclock_s": 150.33362078666687
+}

attribution/pythia-160m/ioi-results.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d4eb1c1dce4716843805b815c9fdef324faf58626fbf14f9ceee0545404188e
+size 32822469

attribution/pythia-160m/ioi-verdict.json

@@ -0,0 +1,11 @@
+{
+  "passed": true,
+  "max_ratio": 167.48154981549814,
+  "ratio_at_k": 64,
+  "ig_clean": true,
+  "n_samples": 1000,
+  "n_ig_steps": 10,
+  "snapshot_step": 143000,
+  "checkpoint": "/home/mh2274/rds/hpc-work/wu_crosscoder/run3_ge_exact_results/wu_cc_ge_exact_seed0.pt",
+  "wallclock_s": 92.33377718925476
+}

attribution/pythia-160m/sva-results.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39c166606642bf4758e30c06d33935ec7bd2a1855adb19a7b8252b59a9850425
+size 32822469

attribution/pythia-160m/sva-verdict.json

@@ -0,0 +1,11 @@
+{
+  "passed": true,
+  "max_ratio": 1140.1538461538462,
+  "ratio_at_k": 64,
+  "ig_clean": true,
+  "n_samples": 1000,
+  "n_ig_steps": 10,
+  "snapshot_step": 143000,
+  "checkpoint": "/home/mh2274/rds/hpc-work/wu_crosscoder/run3_ge_exact_results/wu_cc_ge_exact_seed0.pt",
+  "wallclock_s": 191.5964698791504
+}

pythia-160m/W_E/cross-snapshot-32/d24576/seed0.config.json

@@ -0,0 +1,119 @@
+{
+  "public_label": "Pythia-160M W_E crosscoder, 32 snapshots, d_sae=24576, seed 0",
+  "config": {
+    "sae_type": "crosscoder",
+    "d_model": 768,
+    "expansion_factor": 32.0,
+    "use_decoder_bias": true,
+    "act_fn": "jumprelu",
+    "norm_activation": "inference",
+    "sparsity_include_decoder_norm": true,
+    "top_k": 50,
+    "use_triton_kernel": false,
+    "sparsity_threshold_for_triton_spmm_kernel": 0.996,
+    "jumprelu_threshold_window": 4.0,
+    "hook_points": [
+      "wu_step_0",
+      "wu_step_1",
+      "wu_step_2",
+      "wu_step_3",
+      "wu_step_4",
+      "wu_step_5",
+      "wu_step_6",
+      "wu_step_7",
+      "wu_step_8",
+      "wu_step_9",
+      "wu_step_10",
+      "wu_step_11",
+      "wu_step_12",
+      "wu_step_13",
+      "wu_step_14",
+      "wu_step_15",
+      "wu_step_16",
+      "wu_step_17",
+      "wu_step_18",
+      "wu_step_19",
+      "wu_step_20",
+      "wu_step_21",
+      "wu_step_22",
+      "wu_step_23",
+      "wu_step_24",
+      "wu_step_25",
+      "wu_step_26",
+      "wu_step_27",
+      "wu_step_28",
+      "wu_step_29",
+      "wu_step_30",
+      "wu_step_31"
+    ]
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "frequency_scale": 0.01,
+    "tanh_stretch_coefficient": 4.0,
+    "jumprelu_lr_factor": 0.1,
+    "init_threshold": 0.1,
+    "decoder_transpose_init": 1.0,
+    "amp_dtype": "fp32",
+    "optimizer": "adam",
+    "n_epochs": 300,
+    "batch_size": 1024,
+    "expansion_factor": 32.0,
+    "d_sae": 24576,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.05,
+    "lr_warmup_fraction": 0.0,
+    "lr_decay_fraction": 0.0,
+    "auxk_coefficient": 0.0,
+    "auxk_k": 512,
+    "dead_window_steps": 100
+  },
+  "quality": {
+    "explained_variance": 0.8305442370487662,
+    "reconstruction_mse": 0.1690640303492546,
+    "mean_l0": 117.50901977539063,
+    "dead_rate": 0.0,
+    "d_sae": 24576,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 0,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_E/cross-snapshot-32/d24576/seed0.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_E crosscoder, 32 snapshots, d_sae=24576, seed 0
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 14000, 21000, 27000, 34000, 47000, 61000, 75000, 89000, 102000, 116000, 130000, 143000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.8305442370487662
+- Mean L0: 117.50901977539063
+- Dead-feature rate: 0.0
+- d_sae: 24576
+- n_snapshots: 32
+
+## Training recipe (key hparams)
+- Activation: jumprelu
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 4.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: 0.1
+- Optimizer: adam
+- Epochs: 300
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 0
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed0.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed0.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_E/cross-snapshot-32/d24576/seed0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfe74424d03c865e4595f94a9aff6316d579a52b963255807577233d5b5387f2
+size 4835180992

pythia-160m/W_E/cross-snapshot-32/d8192/seed0.config.json

@@ -0,0 +1,113 @@
+{
+  "public_label": "Pythia-160M W_E crosscoder, 32 snapshots, d_sae=8192, seed 0",
+  "config": {
+    "sae_type": "crosscoder",
+    "d_model": 768,
+    "expansion_factor": 10.667,
+    "use_decoder_bias": true,
+    "act_fn": "jumprelu",
+    "norm_activation": "inference",
+    "sparsity_include_decoder_norm": true,
+    "top_k": 50,
+    "use_triton_kernel": false,
+    "sparsity_threshold_for_triton_spmm_kernel": 0.996,
+    "jumprelu_threshold_window": 4.0,
+    "hook_points": [
+      "wu_step_0",
+      "wu_step_1",
+      "wu_step_2",
+      "wu_step_3",
+      "wu_step_4",
+      "wu_step_5",
+      "wu_step_6",
+      "wu_step_7",
+      "wu_step_8",
+      "wu_step_9",
+      "wu_step_10",
+      "wu_step_11",
+      "wu_step_12",
+      "wu_step_13",
+      "wu_step_14",
+      "wu_step_15",
+      "wu_step_16",
+      "wu_step_17",
+      "wu_step_18",
+      "wu_step_19",
+      "wu_step_20",
+      "wu_step_21",
+      "wu_step_22",
+      "wu_step_23",
+      "wu_step_24",
+      "wu_step_25",
+      "wu_step_26",
+      "wu_step_27",
+      "wu_step_28",
+      "wu_step_29",
+      "wu_step_30",
+      "wu_step_31"
+    ]
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "frequency_scale": 0.01,
+    "tanh_stretch_coefficient": 4.0,
+    "jumprelu_lr_factor": 0.1,
+    "n_epochs": 300,
+    "batch_size": 1024,
+    "expansion_factor": 10.667,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.05,
+    "lr_warmup_fraction": 0.0,
+    "lr_decay_fraction": 0.0,
+    "auxk_coefficient": 0.0,
+    "auxk_k": 512,
+    "dead_window_steps": 100
+  },
+  "quality": {
+    "explained_variance": 0.5807971126242004,
+    "reconstruction_mse": 0.4182338178157806,
+    "mean_l0": 82.08435302734375,
+    "d_sae": 8192,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 0,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_E/cross-snapshot-32/d8192/seed0.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_E crosscoder, 32 snapshots, d_sae=8192, seed 0
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 14000, 21000, 27000, 34000, 47000, 61000, 75000, 89000, 102000, 116000, 130000, 143000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.5807971126242004
+- Mean L0: 82.08435302734375
+- Dead-feature rate: —
+- d_sae: 8192
+- n_snapshots: 32
+
+## Training recipe (key hparams)
+- Activation: jumprelu
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 4.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: —
+- Optimizer: adam
+- Epochs: 300
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 0
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed0.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed0.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_E/cross-snapshot-32/d8192/seed0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75ba91f6abdbd6b888ad7fc8ba34a30a363a123846bc136bbe29bb19386374bb
+size 1611792824

pythia-160m/W_E/cross-snapshot-32/d8192/seed1.config.json

@@ -0,0 +1,119 @@
+{
+  "public_label": "Pythia-160M W_E crosscoder, 32 snapshots, d_sae=8192, seed 1",
+  "config": {
+    "sae_type": "crosscoder",
+    "d_model": 768,
+    "expansion_factor": 10.667,
+    "use_decoder_bias": true,
+    "act_fn": "jumprelu",
+    "norm_activation": "inference",
+    "sparsity_include_decoder_norm": true,
+    "top_k": 50,
+    "use_triton_kernel": false,
+    "sparsity_threshold_for_triton_spmm_kernel": 0.996,
+    "jumprelu_threshold_window": 4.0,
+    "hook_points": [
+      "wu_step_0",
+      "wu_step_1",
+      "wu_step_2",
+      "wu_step_3",
+      "wu_step_4",
+      "wu_step_5",
+      "wu_step_6",
+      "wu_step_7",
+      "wu_step_8",
+      "wu_step_9",
+      "wu_step_10",
+      "wu_step_11",
+      "wu_step_12",
+      "wu_step_13",
+      "wu_step_14",
+      "wu_step_15",
+      "wu_step_16",
+      "wu_step_17",
+      "wu_step_18",
+      "wu_step_19",
+      "wu_step_20",
+      "wu_step_21",
+      "wu_step_22",
+      "wu_step_23",
+      "wu_step_24",
+      "wu_step_25",
+      "wu_step_26",
+      "wu_step_27",
+      "wu_step_28",
+      "wu_step_29",
+      "wu_step_30",
+      "wu_step_31"
+    ]
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "frequency_scale": 0.01,
+    "tanh_stretch_coefficient": 4.0,
+    "jumprelu_lr_factor": 0.1,
+    "init_threshold": 0.1,
+    "decoder_transpose_init": 1.0,
+    "amp_dtype": "fp32",
+    "optimizer": "adam",
+    "n_epochs": 300,
+    "batch_size": 1024,
+    "expansion_factor": 10.667,
+    "d_sae": 8192,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.05,
+    "lr_warmup_fraction": 0.0,
+    "lr_decay_fraction": 0.0,
+    "auxk_coefficient": 0.0,
+    "auxk_k": 512,
+    "dead_window_steps": 100
+  },
+  "quality": {
+    "explained_variance": 0.580104140268709,
+    "reconstruction_mse": 0.41892518222332,
+    "mean_l0": 82.01199279785156,
+    "dead_rate": 0.0,
+    "d_sae": 8192,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 1,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_E/cross-snapshot-32/d8192/seed1.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_E crosscoder, 32 snapshots, d_sae=8192, seed 1
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 14000, 21000, 27000, 34000, 47000, 61000, 75000, 89000, 102000, 116000, 130000, 143000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.580104140268709
+- Mean L0: 82.01199279785156
+- Dead-feature rate: 0.0
+- d_sae: 8192
+- n_snapshots: 32
+
+## Training recipe (key hparams)
+- Activation: jumprelu
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 4.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: 0.1
+- Optimizer: adam
+- Epochs: 300
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 1
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed1.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed1.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_E/cross-snapshot-32/d8192/seed1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:777f45bcec5b17bf9c7d080c79db5741cfdfb358743d307cc0ae69ce8c2fbb96
+size 1611792824

pythia-160m/W_E/cross-snapshot-32/d8192/seed2.config.json

@@ -0,0 +1,119 @@
+{
+  "public_label": "Pythia-160M W_E crosscoder, 32 snapshots, d_sae=8192, seed 2",
+  "config": {
+    "sae_type": "crosscoder",
+    "d_model": 768,
+    "expansion_factor": 10.667,
+    "use_decoder_bias": true,
+    "act_fn": "jumprelu",
+    "norm_activation": "inference",
+    "sparsity_include_decoder_norm": true,
+    "top_k": 50,
+    "use_triton_kernel": false,
+    "sparsity_threshold_for_triton_spmm_kernel": 0.996,
+    "jumprelu_threshold_window": 4.0,
+    "hook_points": [
+      "wu_step_0",
+      "wu_step_1",
+      "wu_step_2",
+      "wu_step_3",
+      "wu_step_4",
+      "wu_step_5",
+      "wu_step_6",
+      "wu_step_7",
+      "wu_step_8",
+      "wu_step_9",
+      "wu_step_10",
+      "wu_step_11",
+      "wu_step_12",
+      "wu_step_13",
+      "wu_step_14",
+      "wu_step_15",
+      "wu_step_16",
+      "wu_step_17",
+      "wu_step_18",
+      "wu_step_19",
+      "wu_step_20",
+      "wu_step_21",
+      "wu_step_22",
+      "wu_step_23",
+      "wu_step_24",
+      "wu_step_25",
+      "wu_step_26",
+      "wu_step_27",
+      "wu_step_28",
+      "wu_step_29",
+      "wu_step_30",
+      "wu_step_31"
+    ]
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "frequency_scale": 0.01,
+    "tanh_stretch_coefficient": 4.0,
+    "jumprelu_lr_factor": 0.1,
+    "init_threshold": 0.1,
+    "decoder_transpose_init": 1.0,
+    "amp_dtype": "fp32",
+    "optimizer": "adam",
+    "n_epochs": 300,
+    "batch_size": 1024,
+    "expansion_factor": 10.667,
+    "d_sae": 8192,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.05,
+    "lr_warmup_fraction": 0.0,
+    "lr_decay_fraction": 0.0,
+    "auxk_coefficient": 0.0,
+    "auxk_k": 512,
+    "dead_window_steps": 100
+  },
+  "quality": {
+    "explained_variance": 0.5815380601296635,
+    "reconstruction_mse": 0.41749457716941835,
+    "mean_l0": 82.31076721191407,
+    "dead_rate": 0.0,
+    "d_sae": 8192,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 2,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_E/cross-snapshot-32/d8192/seed2.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_E crosscoder, 32 snapshots, d_sae=8192, seed 2
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 14000, 21000, 27000, 34000, 47000, 61000, 75000, 89000, 102000, 116000, 130000, 143000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.5815380601296635
+- Mean L0: 82.31076721191407
+- Dead-feature rate: 0.0
+- d_sae: 8192
+- n_snapshots: 32
+
+## Training recipe (key hparams)
+- Activation: jumprelu
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 4.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: 0.1
+- Optimizer: adam
+- Epochs: 300
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 2
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed2.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed2.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_E/cross-snapshot-32/d8192/seed2.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33cee54d608967373984411fffdff2da1bf39aca2a46e14ba75620a82b65e053
+size 1611792824

pythia-160m/W_E/cross-snapshot-32/d8192/seed3.config.json

@@ -0,0 +1,119 @@
+{
+  "public_label": "Pythia-160M W_E crosscoder, 32 snapshots, d_sae=8192, seed 3",
+  "config": {
+    "sae_type": "crosscoder",
+    "d_model": 768,
+    "expansion_factor": 10.667,
+    "use_decoder_bias": true,
+    "act_fn": "jumprelu",
+    "norm_activation": "inference",
+    "sparsity_include_decoder_norm": true,
+    "top_k": 50,
+    "use_triton_kernel": false,
+    "sparsity_threshold_for_triton_spmm_kernel": 0.996,
+    "jumprelu_threshold_window": 4.0,
+    "hook_points": [
+      "wu_step_0",
+      "wu_step_1",
+      "wu_step_2",
+      "wu_step_3",
+      "wu_step_4",
+      "wu_step_5",
+      "wu_step_6",
+      "wu_step_7",
+      "wu_step_8",
+      "wu_step_9",
+      "wu_step_10",
+      "wu_step_11",
+      "wu_step_12",
+      "wu_step_13",
+      "wu_step_14",
+      "wu_step_15",
+      "wu_step_16",
+      "wu_step_17",
+      "wu_step_18",
+      "wu_step_19",
+      "wu_step_20",
+      "wu_step_21",
+      "wu_step_22",
+      "wu_step_23",
+      "wu_step_24",
+      "wu_step_25",
+      "wu_step_26",
+      "wu_step_27",
+      "wu_step_28",
+      "wu_step_29",
+      "wu_step_30",
+      "wu_step_31"
+    ]
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "frequency_scale": 0.01,
+    "tanh_stretch_coefficient": 4.0,
+    "jumprelu_lr_factor": 0.1,
+    "init_threshold": 0.1,
+    "decoder_transpose_init": 1.0,
+    "amp_dtype": "fp32",
+    "optimizer": "adam",
+    "n_epochs": 300,
+    "batch_size": 1024,
+    "expansion_factor": 10.667,
+    "d_sae": 8192,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.05,
+    "lr_warmup_fraction": 0.0,
+    "lr_decay_fraction": 0.0,
+    "auxk_coefficient": 0.0,
+    "auxk_k": 512,
+    "dead_window_steps": 100
+  },
+  "quality": {
+    "explained_variance": 0.5805984141666259,
+    "reconstruction_mse": 0.41843205094337466,
+    "mean_l0": 82.13521545410157,
+    "dead_rate": 0.0,
+    "d_sae": 8192,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 3,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_E/cross-snapshot-32/d8192/seed3.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_E crosscoder, 32 snapshots, d_sae=8192, seed 3
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 14000, 21000, 27000, 34000, 47000, 61000, 75000, 89000, 102000, 116000, 130000, 143000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.5805984141666259
+- Mean L0: 82.13521545410157
+- Dead-feature rate: 0.0
+- d_sae: 8192
+- n_snapshots: 32
+
+## Training recipe (key hparams)
+- Activation: jumprelu
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 4.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: 0.1
+- Optimizer: adam
+- Epochs: 300
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 3
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed3.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed3.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_E/cross-snapshot-32/d8192/seed3.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f54af8a2712eafc61e6c7ba219e88f36ccd21b03abaec4661b7e139753acf0bc
+size 1611792824

pythia-160m/W_E/cross-snapshot-32/d8192/seed4.config.json

@@ -0,0 +1,119 @@
+{
+  "public_label": "Pythia-160M W_E crosscoder, 32 snapshots, d_sae=8192, seed 4",
+  "config": {
+    "sae_type": "crosscoder",
+    "d_model": 768,
+    "expansion_factor": 10.667,
+    "use_decoder_bias": true,
+    "act_fn": "jumprelu",
+    "norm_activation": "inference",
+    "sparsity_include_decoder_norm": true,
+    "top_k": 50,
+    "use_triton_kernel": false,
+    "sparsity_threshold_for_triton_spmm_kernel": 0.996,
+    "jumprelu_threshold_window": 4.0,
+    "hook_points": [
+      "wu_step_0",
+      "wu_step_1",
+      "wu_step_2",
+      "wu_step_3",
+      "wu_step_4",
+      "wu_step_5",
+      "wu_step_6",
+      "wu_step_7",
+      "wu_step_8",
+      "wu_step_9",
+      "wu_step_10",
+      "wu_step_11",
+      "wu_step_12",
+      "wu_step_13",
+      "wu_step_14",
+      "wu_step_15",
+      "wu_step_16",
+      "wu_step_17",
+      "wu_step_18",
+      "wu_step_19",
+      "wu_step_20",
+      "wu_step_21",
+      "wu_step_22",
+      "wu_step_23",
+      "wu_step_24",
+      "wu_step_25",
+      "wu_step_26",
+      "wu_step_27",
+      "wu_step_28",
+      "wu_step_29",
+      "wu_step_30",
+      "wu_step_31"
+    ]
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "frequency_scale": 0.01,
+    "tanh_stretch_coefficient": 4.0,
+    "jumprelu_lr_factor": 0.1,
+    "init_threshold": 0.1,
+    "decoder_transpose_init": 1.0,
+    "amp_dtype": "fp32",
+    "optimizer": "adam",
+    "n_epochs": 300,
+    "batch_size": 1024,
+    "expansion_factor": 10.667,
+    "d_sae": 8192,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.05,
+    "lr_warmup_fraction": 0.0,
+    "lr_decay_fraction": 0.0,
+    "auxk_coefficient": 0.0,
+    "auxk_k": 512,
+    "dead_window_steps": 100
+  },
+  "quality": {
+    "explained_variance": 0.5825219599393371,
+    "reconstruction_mse": 0.4165129518508911,
+    "mean_l0": 82.5084307861328,
+    "dead_rate": 0.0,
+    "d_sae": 8192,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 4,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_E/cross-snapshot-32/d8192/seed4.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_E crosscoder, 32 snapshots, d_sae=8192, seed 4
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 14000, 21000, 27000, 34000, 47000, 61000, 75000, 89000, 102000, 116000, 130000, 143000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.5825219599393371
+- Mean L0: 82.5084307861328
+- Dead-feature rate: 0.0
+- d_sae: 8192
+- n_snapshots: 32
+
+## Training recipe (key hparams)
+- Activation: jumprelu
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 4.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: 0.1
+- Optimizer: adam
+- Epochs: 300
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 4
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed4.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed4.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_E/cross-snapshot-32/d8192/seed4.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712affee5958492d3f114b48e3f718a8b366513fab63b4d48a9b8f4c0dcf753b
+size 1611792824

pythia-160m/W_U/architecture-comparison/d8192/batchtopk/seed0.config.json

@@ -0,0 +1,74 @@
+{
+  "public_label": "Pythia-160M W_U crosscoder (batchtopk activation), 32 snapshots, d_sae=8192, seed 0",
+  "config": {
+    "arch": "batchtopk",
+    "d_sae": 8192,
+    "n_snapshots": 32,
+    "d_model": 768,
+    "batchtopk_k": 203
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "gated_l1_coefficient": 0.3,
+    "init_threshold": 0.1,
+    "bandwidth": 4.0,
+    "frequency_scale": 0.01,
+    "tanh_stretch": 1.0,
+    "jumprelu_lr_factor": 0.1,
+    "n_epochs": 100,
+    "batch_size": 1024,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.1,
+    "lr_warmup_fraction": 0.0,
+    "lr_decay_fraction": 0.0,
+    "git_hash": "unknown",
+    "amp_dtype": "fp32"
+  },
+  "quality": {
+    "explained_variance": 0.7249162974116965,
+    "reconstruction_mse": 0.2749960619211197,
+    "mean_l0": 203.0,
+    "d_sae": 8192,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 0,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_U/architecture-comparison/d8192/batchtopk/seed0.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_U crosscoder (batchtopk activation), 32 snapshots, d_sae=8192, seed 0
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 14000, 21000, 27000, 34000, 47000, 61000, 75000, 89000, 102000, 116000, 130000, 143000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.7249162974116965
+- Mean L0: 203.0
+- Dead-feature rate: —
+- d_sae: 8192
+- n_snapshots: 32
+
+## Training recipe (key hparams)
+- Activation: JumpReLU
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 1.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: 0.1
+- Optimizer: adam
+- Epochs: 100
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 0
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed0.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed0.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_U/architecture-comparison/d8192/batchtopk/seed0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9e434afc4191e631e88fc31a1ff33068b24a26c6650066b8ec67e463ec0c790
+size 1611759944

pythia-160m/W_U/architecture-comparison/d8192/gated-retuned/seed0.config.json

@@ -0,0 +1,74 @@
+{
+  "public_label": "Pythia-160M W_U crosscoder (Gated activation, L1=0.05), 32 snapshots, d_sae=8192, seed 0",
+  "config": {
+    "arch": "gated",
+    "d_sae": 8192,
+    "n_snapshots": 32,
+    "d_model": 768,
+    "batchtopk_k": null
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "gated_l1_coefficient": 0.05,
+    "init_threshold": 0.1,
+    "bandwidth": 4.0,
+    "frequency_scale": 0.01,
+    "tanh_stretch": 1.0,
+    "jumprelu_lr_factor": 0.1,
+    "n_epochs": 100,
+    "batch_size": 1024,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.1,
+    "lr_warmup_fraction": 0.0,
+    "lr_decay_fraction": 0.0,
+    "git_hash": "unknown",
+    "amp_dtype": "fp32"
+  },
+  "quality": {
+    "explained_variance": 0.827462127951851,
+    "reconstruction_mse": 0.17248290210962294,
+    "mean_l0": 653.64568359375,
+    "d_sae": 8192,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 0,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_U/architecture-comparison/d8192/gated-retuned/seed0.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_U crosscoder (Gated activation, L1=0.05), 32 snapshots, d_sae=8192, seed 0
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 14000, 21000, 27000, 34000, 47000, 61000, 75000, 89000, 102000, 116000, 130000, 143000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.827462127951851
+- Mean L0: 653.64568359375
+- Dead-feature rate: —
+- d_sae: 8192
+- n_snapshots: 32
+
+## Training recipe (key hparams)
+- Activation: JumpReLU
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 1.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: 0.1
+- Optimizer: adam
+- Epochs: 100
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 0
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed0.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed0.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_U/architecture-comparison/d8192/gated-retuned/seed0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:197aaa8bcccd820a827454ad03ae72d0f3306b840719a7251d96e6bf8f4c4120
+size 1611858536

pythia-160m/W_U/architecture-comparison/d8192/gated/seed0.config.json

@@ -0,0 +1,74 @@
+{
+  "public_label": "Pythia-160M W_U crosscoder (gated activation), 32 snapshots, d_sae=8192, seed 0",
+  "config": {
+    "arch": "gated",
+    "d_sae": 8192,
+    "n_snapshots": 32,
+    "d_model": 768,
+    "batchtopk_k": null
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "gated_l1_coefficient": 0.3,
+    "init_threshold": 0.1,
+    "bandwidth": 4.0,
+    "frequency_scale": 0.01,
+    "tanh_stretch": 1.0,
+    "jumprelu_lr_factor": 0.1,
+    "n_epochs": 100,
+    "batch_size": 1024,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.1,
+    "lr_warmup_fraction": 0.0,
+    "lr_decay_fraction": 0.0,
+    "git_hash": "unknown",
+    "amp_dtype": "fp32"
+  },
+  "quality": {
+    "explained_variance": 0.21442979106557436,
+    "reconstruction_mse": 0.7853199291229248,
+    "mean_l0": 11.51853515625,
+    "d_sae": 8192,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 0,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_U/architecture-comparison/d8192/gated/seed0.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_U crosscoder (gated activation), 32 snapshots, d_sae=8192, seed 0
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 14000, 21000, 27000, 34000, 47000, 61000, 75000, 89000, 102000, 116000, 130000, 143000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.21442979106557436
+- Mean L0: 11.51853515625
+- Dead-feature rate: —
+- d_sae: 8192
+- n_snapshots: 32
+
+## Training recipe (key hparams)
+- Activation: JumpReLU
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 1.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: 0.1
+- Optimizer: adam
+- Epochs: 100
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 0
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed0.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed0.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_U/architecture-comparison/d8192/gated/seed0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10b27d935481b43900bab227acea5be346d403e163d2976abb3dc0676adde955
+size 1611858536

pythia-160m/W_U/cross-snapshot-16/d8192/seed0.config.json

@@ -0,0 +1,87 @@
+{
+  "public_label": "Pythia-160M W_U crosscoder, 16-snapshot downsample, d_sae=8192, seed 0",
+  "config": {
+    "sae_type": "crosscoder",
+    "d_model": 768,
+    "expansion_factor": 10.6667,
+    "use_decoder_bias": true,
+    "act_fn": "jumprelu",
+    "norm_activation": "inference",
+    "sparsity_include_decoder_norm": true,
+    "top_k": 50,
+    "use_triton_kernel": false,
+    "sparsity_threshold_for_triton_spmm_kernel": 0.996,
+    "jumprelu_threshold_window": 4.0,
+    "hook_points": [
+      "wu_step_0",
+      "wu_step_1",
+      "wu_step_2",
+      "wu_step_3",
+      "wu_step_4",
+      "wu_step_5",
+      "wu_step_6",
+      "wu_step_7",
+      "wu_step_8",
+      "wu_step_9",
+      "wu_step_10",
+      "wu_step_11",
+      "wu_step_12",
+      "wu_step_13",
+      "wu_step_14",
+      "wu_step_15"
+    ]
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "frequency_scale": 0.01,
+    "tanh_stretch_coefficient": 1.0,
+    "jumprelu_lr_factor": 0.1,
+    "init_threshold": 0.1,
+    "decoder_transpose_init": 1.0,
+    "amp_dtype": "fp32",
+    "optimizer": "adam",
+    "n_epochs": 300,
+    "batch_size": 1024,
+    "expansion_factor": 10.6667,
+    "d_sae": 8192,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.05,
+    "lr_warmup_fraction": 0.0,
+    "lr_decay_fraction": 0.0,
+    "auxk_coefficient": 0.0,
+    "auxk_k": 512,
+    "dead_window_steps": 100
+  },
+  "quality": {
+    "explained_variance": 0.7734329300485889,
+    "reconstruction_mse": 0.22638385832309724,
+    "mean_l0": 215.73649780273436,
+    "dead_rate": 0.0,
+    "d_sae": 8192,
+    "n_snapshots": 16,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    2,
+    8,
+    32,
+    128,
+    512,
+    2000,
+    4000,
+    6000,
+    8000,
+    14000,
+    27000,
+    47000,
+    75000,
+    102000,
+    130000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 0,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_U/cross-snapshot-16/d8192/seed0.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_U crosscoder, 16-snapshot downsample, d_sae=8192, seed 0
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 2, 8, 32, 128, 512, 2000, 4000, 6000, 8000, 14000, 27000, 47000, 75000, 102000, 130000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.7734329300485889
+- Mean L0: 215.73649780273436
+- Dead-feature rate: 0.0
+- d_sae: 8192
+- n_snapshots: 16
+
+## Training recipe (key hparams)
+- Activation: jumprelu
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 1.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: 0.1
+- Optimizer: adam
+- Epochs: 300
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 0
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed0.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed0.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_U/cross-snapshot-16/d8192/seed0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8496ae3b72b23b0a65698e3e424eddc411bd6ac919f1aaa915ae06a147b3803a
+size 805913008

pythia-160m/W_U/cross-snapshot-32/d16384/seed0.config.json

@@ -0,0 +1,111 @@
+{
+  "public_label": "Pythia-160M W_U crosscoder, 32 snapshots, d_sae=16384, seed 0",
+  "config": {
+    "sae_type": "crosscoder",
+    "d_model": 768,
+    "expansion_factor": 21.334,
+    "use_decoder_bias": true,
+    "act_fn": "jumprelu",
+    "norm_activation": "inference",
+    "sparsity_include_decoder_norm": true,
+    "top_k": 50,
+    "use_triton_kernel": false,
+    "sparsity_threshold_for_triton_spmm_kernel": 0.996,
+    "jumprelu_threshold_window": 4.0,
+    "hook_points": [
+      "wu_step_0",
+      "wu_step_1",
+      "wu_step_2",
+      "wu_step_3",
+      "wu_step_4",
+      "wu_step_5",
+      "wu_step_6",
+      "wu_step_7",
+      "wu_step_8",
+      "wu_step_9",
+      "wu_step_10",
+      "wu_step_11",
+      "wu_step_12",
+      "wu_step_13",
+      "wu_step_14",
+      "wu_step_15",
+      "wu_step_16",
+      "wu_step_17",
+      "wu_step_18",
+      "wu_step_19",
+      "wu_step_20",
+      "wu_step_21",
+      "wu_step_22",
+      "wu_step_23",
+      "wu_step_24",
+      "wu_step_25",
+      "wu_step_26",
+      "wu_step_27",
+      "wu_step_28",
+      "wu_step_29",
+      "wu_step_30",
+      "wu_step_31"
+    ]
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "frequency_scale": 0.01,
+    "tanh_stretch_coefficient": 4.0,
+    "jumprelu_lr_factor": 0.1,
+    "n_epochs": 300,
+    "batch_size": 1024,
+    "expansion_factor": 21.334,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.05,
+    "auxk_coefficient": 0.0,
+    "auxk_k": 512,
+    "dead_window_steps": 100
+  },
+  "quality": {
+    "explained_variance": 0.7802076016978367,
+    "reconstruction_mse": 0.21972236931324005,
+    "mean_l0": 103.0708544921875,
+    "d_sae": 16384,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 0,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_U/cross-snapshot-32/d16384/seed0.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_U crosscoder, 32 snapshots, d_sae=16384, seed 0
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 14000, 21000, 27000, 34000, 47000, 61000, 75000, 89000, 102000, 116000, 130000, 143000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.7802076016978367
+- Mean L0: 103.0708544921875
+- Dead-feature rate: —
+- d_sae: 16384
+- n_snapshots: 32
+
+## Training recipe (key hparams)
+- Activation: jumprelu
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 4.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: —
+- Optimizer: adam
+- Epochs: 300
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 0
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed0.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed0.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_U/cross-snapshot-32/d16384/seed0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a9b590ef230642ca44ea8a562155785aacd314bc757a845fc88cb3e12f72af1
+size 3223486912

pythia-160m/W_U/cross-snapshot-32/d24576/seed0.config.json

@@ -0,0 +1,119 @@
+{
+  "public_label": "Pythia-160M W_U crosscoder, 32 snapshots, d_sae=24576, seed 0",
+  "config": {
+    "sae_type": "crosscoder",
+    "d_model": 768,
+    "expansion_factor": 32.0,
+    "use_decoder_bias": true,
+    "act_fn": "jumprelu",
+    "norm_activation": "inference",
+    "sparsity_include_decoder_norm": true,
+    "top_k": 50,
+    "use_triton_kernel": false,
+    "sparsity_threshold_for_triton_spmm_kernel": 0.996,
+    "jumprelu_threshold_window": 4.0,
+    "hook_points": [
+      "wu_step_0",
+      "wu_step_1",
+      "wu_step_2",
+      "wu_step_3",
+      "wu_step_4",
+      "wu_step_5",
+      "wu_step_6",
+      "wu_step_7",
+      "wu_step_8",
+      "wu_step_9",
+      "wu_step_10",
+      "wu_step_11",
+      "wu_step_12",
+      "wu_step_13",
+      "wu_step_14",
+      "wu_step_15",
+      "wu_step_16",
+      "wu_step_17",
+      "wu_step_18",
+      "wu_step_19",
+      "wu_step_20",
+      "wu_step_21",
+      "wu_step_22",
+      "wu_step_23",
+      "wu_step_24",
+      "wu_step_25",
+      "wu_step_26",
+      "wu_step_27",
+      "wu_step_28",
+      "wu_step_29",
+      "wu_step_30",
+      "wu_step_31"
+    ]
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "frequency_scale": 0.01,
+    "tanh_stretch_coefficient": 1.0,
+    "jumprelu_lr_factor": 0.1,
+    "init_threshold": 0.1,
+    "decoder_transpose_init": 1.0,
+    "amp_dtype": "fp32",
+    "optimizer": "adam",
+    "n_epochs": 300,
+    "batch_size": 1024,
+    "expansion_factor": 32.0,
+    "d_sae": 24576,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.05,
+    "lr_warmup_fraction": 0.0,
+    "lr_decay_fraction": 0.0,
+    "auxk_coefficient": 0.0,
+    "auxk_k": 512,
+    "dead_window_steps": 100
+  },
+  "quality": {
+    "explained_variance": 0.9196719747097695,
+    "reconstruction_mse": 0.08030243307352065,
+    "mean_l0": 285.9946594238281,
+    "dead_rate": 0.0,
+    "d_sae": 24576,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 0,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_U/cross-snapshot-32/d24576/seed0.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_U crosscoder, 32 snapshots, d_sae=24576, seed 0
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 14000, 21000, 27000, 34000, 47000, 61000, 75000, 89000, 102000, 116000, 130000, 143000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.9196719747097695
+- Mean L0: 285.9946594238281
+- Dead-feature rate: 0.0
+- d_sae: 24576
+- n_snapshots: 32
+
+## Training recipe (key hparams)
+- Activation: jumprelu
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 1.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: 0.1
+- Optimizer: adam
+- Epochs: 300
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 0
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed0.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed0.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_U/cross-snapshot-32/d24576/seed0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5998eba9b1f0f94f7579c692dcee2de8a2875f2f80f3fb4e127f6024f411e815
+size 4835180992

pythia-160m/W_U/cross-snapshot-32/d24576/seed1.config.json

@@ -0,0 +1,119 @@
+{
+  "public_label": "Pythia-160M W_U crosscoder, 32 snapshots, d_sae=24576, seed 1",
+  "config": {
+    "sae_type": "crosscoder",
+    "d_model": 768,
+    "expansion_factor": 32.0,
+    "use_decoder_bias": true,
+    "act_fn": "jumprelu",
+    "norm_activation": "inference",
+    "sparsity_include_decoder_norm": true,
+    "top_k": 50,
+    "use_triton_kernel": false,
+    "sparsity_threshold_for_triton_spmm_kernel": 0.996,
+    "jumprelu_threshold_window": 4.0,
+    "hook_points": [
+      "wu_step_0",
+      "wu_step_1",
+      "wu_step_2",
+      "wu_step_3",
+      "wu_step_4",
+      "wu_step_5",
+      "wu_step_6",
+      "wu_step_7",
+      "wu_step_8",
+      "wu_step_9",
+      "wu_step_10",
+      "wu_step_11",
+      "wu_step_12",
+      "wu_step_13",
+      "wu_step_14",
+      "wu_step_15",
+      "wu_step_16",
+      "wu_step_17",
+      "wu_step_18",
+      "wu_step_19",
+      "wu_step_20",
+      "wu_step_21",
+      "wu_step_22",
+      "wu_step_23",
+      "wu_step_24",
+      "wu_step_25",
+      "wu_step_26",
+      "wu_step_27",
+      "wu_step_28",
+      "wu_step_29",
+      "wu_step_30",
+      "wu_step_31"
+    ]
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "frequency_scale": 0.01,
+    "tanh_stretch_coefficient": 1.0,
+    "jumprelu_lr_factor": 0.1,
+    "init_threshold": 0.1,
+    "decoder_transpose_init": 1.0,
+    "amp_dtype": "fp32",
+    "optimizer": "adam",
+    "n_epochs": 300,
+    "batch_size": 1024,
+    "expansion_factor": 32.0,
+    "d_sae": 24576,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.05,
+    "lr_warmup_fraction": 0.0,
+    "lr_decay_fraction": 0.0,
+    "auxk_coefficient": 0.0,
+    "auxk_k": 512,
+    "dead_window_steps": 100
+  },
+  "quality": {
+    "explained_variance": 0.9195844996740984,
+    "reconstruction_mse": 0.08038988023996353,
+    "mean_l0": 286.09990661621094,
+    "dead_rate": 0.0,
+    "d_sae": 24576,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 1,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_U/cross-snapshot-32/d24576/seed1.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_U crosscoder, 32 snapshots, d_sae=24576, seed 1
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 14000, 21000, 27000, 34000, 47000, 61000, 75000, 89000, 102000, 116000, 130000, 143000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.9195844996740984
+- Mean L0: 286.09990661621094
+- Dead-feature rate: 0.0
+- d_sae: 24576
+- n_snapshots: 32
+
+## Training recipe (key hparams)
+- Activation: jumprelu
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 1.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: 0.1
+- Optimizer: adam
+- Epochs: 300
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 1
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed1.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed1.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_U/cross-snapshot-32/d24576/seed1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c071477b1fbd48eb9033488e6943f35e6b9eba95a7839dfc3052701e1ceca4ff
+size 4835180992

pythia-160m/W_U/cross-snapshot-32/d24576/seed2.config.json

@@ -0,0 +1,119 @@
+{
+  "public_label": "Pythia-160M W_U crosscoder, 32 snapshots, d_sae=24576, seed 2",
+  "config": {
+    "sae_type": "crosscoder",
+    "d_model": 768,
+    "expansion_factor": 32.0,
+    "use_decoder_bias": true,
+    "act_fn": "jumprelu",
+    "norm_activation": "inference",
+    "sparsity_include_decoder_norm": true,
+    "top_k": 50,
+    "use_triton_kernel": false,
+    "sparsity_threshold_for_triton_spmm_kernel": 0.996,
+    "jumprelu_threshold_window": 4.0,
+    "hook_points": [
+      "wu_step_0",
+      "wu_step_1",
+      "wu_step_2",
+      "wu_step_3",
+      "wu_step_4",
+      "wu_step_5",
+      "wu_step_6",
+      "wu_step_7",
+      "wu_step_8",
+      "wu_step_9",
+      "wu_step_10",
+      "wu_step_11",
+      "wu_step_12",
+      "wu_step_13",
+      "wu_step_14",
+      "wu_step_15",
+      "wu_step_16",
+      "wu_step_17",
+      "wu_step_18",
+      "wu_step_19",
+      "wu_step_20",
+      "wu_step_21",
+      "wu_step_22",
+      "wu_step_23",
+      "wu_step_24",
+      "wu_step_25",
+      "wu_step_26",
+      "wu_step_27",
+      "wu_step_28",
+      "wu_step_29",
+      "wu_step_30",
+      "wu_step_31"
+    ]
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "frequency_scale": 0.01,
+    "tanh_stretch_coefficient": 1.0,
+    "jumprelu_lr_factor": 0.1,
+    "init_threshold": 0.1,
+    "decoder_transpose_init": 1.0,
+    "amp_dtype": "fp32",
+    "optimizer": "adam",
+    "n_epochs": 300,
+    "batch_size": 1024,
+    "expansion_factor": 32.0,
+    "d_sae": 24576,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.05,
+    "lr_warmup_fraction": 0.0,
+    "lr_decay_fraction": 0.0,
+    "auxk_coefficient": 0.0,
+    "auxk_k": 512,
+    "dead_window_steps": 100
+  },
+  "quality": {
+    "explained_variance": 0.9196765967343927,
+    "reconstruction_mse": 0.08029781252145768,
+    "mean_l0": 286.3446789550781,
+    "dead_rate": 0.0,
+    "d_sae": 24576,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 2,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}

pythia-160m/W_U/cross-snapshot-32/d24576/seed2.md

@@ -0,0 +1,38 @@
+# Pythia-160M W_U crosscoder, 32 snapshots, d_sae=24576, seed 2
+
+Companion artifact to *Parameter-trajectory crosscoders for vocabulary readout
+evolution* (NeurIPS 2026). Trained from snapshots of
+`EleutherAI/pythia-160m` across [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 14000, 21000, 27000, 34000, 47000, 61000, 75000, 89000, 102000, 116000, 130000, 143000] pretraining
+steps.
+
+## Quality
+- Explained variance: 0.9196765967343927
+- Mean L0: 286.3446789550781
+- Dead-feature rate: 0.0
+- d_sae: 24576
+- n_snapshots: 32
+
+## Training recipe (key hparams)
+- Activation: jumprelu
+- L1 coefficient: 0.3
+- Frequency penalty scale: 0.01
+- Tanh-stretch coefficient: 1.0
+- LR: 5e-05
+- JumpReLU LR multiplier: 0.1
+- Init threshold: 0.1
+- Optimizer: adam
+- Epochs: 300
+- Batch size: 1024
+- Preprocess: center_scale
+- Seed: 2
+
+## Loading
+```python
+from safetensors.torch import load_file
+weights = load_file("seed2.safetensors")
+# weights['W_E'] / weights['W_D'] / weights['b_E'] / weights['b_D'] / etc.
+import json
+config = json.load(open("seed2.config.json"))
+```
+
+See the repo top-level README for the full crosscoder loading example.

pythia-160m/W_U/cross-snapshot-32/d24576/seed2.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63a4a1630f355333053e48278a4c4cb5e6616e1ab92700e7d121cb8960083abf
+size 4835180992

pythia-160m/W_U/cross-snapshot-32/d8192/seed0.config.json

@@ -0,0 +1,111 @@
+{
+  "public_label": "Pythia-160M W_U crosscoder, 32 snapshots, d_sae=8192, seed 0",
+  "config": {
+    "sae_type": "crosscoder",
+    "d_model": 768,
+    "expansion_factor": 10.6667,
+    "use_decoder_bias": true,
+    "act_fn": "jumprelu",
+    "norm_activation": "inference",
+    "sparsity_include_decoder_norm": true,
+    "top_k": 50,
+    "use_triton_kernel": false,
+    "sparsity_threshold_for_triton_spmm_kernel": 0.996,
+    "jumprelu_threshold_window": 4.0,
+    "hook_points": [
+      "wu_step_0",
+      "wu_step_1",
+      "wu_step_2",
+      "wu_step_3",
+      "wu_step_4",
+      "wu_step_5",
+      "wu_step_6",
+      "wu_step_7",
+      "wu_step_8",
+      "wu_step_9",
+      "wu_step_10",
+      "wu_step_11",
+      "wu_step_12",
+      "wu_step_13",
+      "wu_step_14",
+      "wu_step_15",
+      "wu_step_16",
+      "wu_step_17",
+      "wu_step_18",
+      "wu_step_19",
+      "wu_step_20",
+      "wu_step_21",
+      "wu_step_22",
+      "wu_step_23",
+      "wu_step_24",
+      "wu_step_25",
+      "wu_step_26",
+      "wu_step_27",
+      "wu_step_28",
+      "wu_step_29",
+      "wu_step_30",
+      "wu_step_31"
+    ]
+  },
+  "training": {
+    "lr": 5e-05,
+    "l1_coefficient": 0.3,
+    "frequency_scale": 0.01,
+    "tanh_stretch_coefficient": 1.0,
+    "jumprelu_lr_factor": 0.1,
+    "n_epochs": 300,
+    "batch_size": 1024,
+    "expansion_factor": 10.6667,
+    "input_preprocess": "center_scale",
+    "l1_warmup_fraction": 0.1,
+    "auxk_coefficient": 0.0,
+    "auxk_k": 512,
+    "dead_window_steps": 100
+  },
+  "quality": {
+    "explained_variance": 0.7763740924044573,
+    "reconstruction_mse": 0.22355465799570085,
+    "mean_l0": 203.14466369628906,
+    "d_sae": 8192,
+    "n_snapshots": 32,
+    "n_rows": 50304
+  },
+  "steps": [
+    0,
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+    256,
+    512,
+    1000,
+    2000,
+    3000,
+    4000,
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    14000,
+    21000,
+    27000,
+    34000,
+    47000,
+    61000,
+    75000,
+    89000,
+    102000,
+    116000,
+    130000,
+    143000
+  ],
+  "model_name": "EleutherAI/pythia-160m",
+  "seed": 0,
+  "kind": "cross-snapshot",
+  "preprocess_mode": "center_scale"
+}