File size: 1,612 Bytes
6b3c76f c03a82b 6b3c76f c03a82b 6b3c76f c03a82b 6b3c76f c03a82b 6b3c76f c03a82b 6b3c76f c03a82b 6b3c76f c03a82b 6b3c76f c03a82b 6b3c76f c03a82b 6b3c76f c03a82b 6b3c76f c03a82b 6b3c76f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | base_save_dir: .checkpoints
cache_dir: .cache
crosscoder:
dec_init_norm: 0.1
ft_init_checkpt_epoch: 0
ft_init_checkpt_folder: null
ft_init_wandb_run_id: ucmhvii8
hidden_dim: 1536
k: 20
data:
activations_harvester:
harvesting_batch_size: 16
inference_dtype: float32
llms:
- lora_name: mars-jason-25/tiny-stories-33M-TSdata-sleeper
name: roneneldan/TinyStories-Instruct-33M
revision: null
activations_shuffle_buffer_size: 1024
n_batches_for_mean_estimate: 100
n_batches_for_norm_estimate: 100
sequence_iterator:
classname: SleeperTokenSequenceLoader
kwargs:
include_sleeper_data: false
sequence_length: 128
shuffle_buffer_size: 1024
experiment_name: lambda_n0_beta_n1_S_2025-07-30_15-10-00
hookpoints:
- blocks.0.hook_resid_pre
- blocks.0.ln1.hook_normalized
- blocks.0.hook_resid_mid
- blocks.0.ln2.hook_normalized
- blocks.1.hook_resid_pre
- blocks.1.ln1.hook_normalized
- blocks.1.hook_resid_mid
- blocks.1.ln2.hook_normalized
- blocks.2.hook_resid_pre
- blocks.2.ln1.hook_normalized
- blocks.2.hook_resid_mid
- blocks.2.ln2.hook_normalized
- blocks.3.hook_resid_pre
- blocks.3.ln1.hook_normalized
- blocks.3.hook_resid_mid
- blocks.3.ln2.hook_normalized
- blocks.3.hook_resid_post
seed: 42
train:
auxK_lambda: 1.0
batch_size: 256
beta_n: 1.0
epochs: null
lam_n: 0.0
log_every_n_steps: 100
num_steps: 10000
num_steps_per_epoch: null
optimizer:
initial_learning_rate: 0.0001
last_pct_of_steps: 0.25
warmup_pct: 0.05
save_every_n_steps: 9999
wandb:
entity: dmitry2-uiuc
project: sleeper-model-diffing
|