| { |
| "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", |
| "sae": { |
| "d_in": 768, |
| "d_sae": 16384, |
| "dtype": "float32", |
| "device": "cuda", |
| "apply_b_dec_to_input": true, |
| "normalize_activations": "none", |
| "reshape_activations": "none", |
| "metadata": { |
| "sae_lens_version": "6.43.0", |
| "sae_lens_training_version": "6.43.0" |
| }, |
| "decoder_init_norm": 0.1, |
| "l1_coefficient": 5.0, |
| "lp_norm": 1.0, |
| "l1_warm_up_steps": 5000, |
| "architecture": "standard" |
| }, |
| "training_samples": 50000000, |
| "batch_size": 4096, |
| "lr": 0.0003, |
| "lr_warm_up_steps": 1000, |
| "lr_decay_steps": 0, |
| "lr_scheduler_name": "constant", |
| "lr_end": 2.9999999999999997e-05, |
| "adam_beta1": 0.9, |
| "adam_beta2": 0.999, |
| "n_restart_cycles": 1, |
| "device": "cuda", |
| "autocast_sae": false, |
| "autocast_data": false, |
| "n_checkpoints": 0, |
| "checkpoint_path": "runs/l1/ckpts", |
| "save_final_checkpoint": true, |
| "output_path": "runs/l1", |
| "save_synthetic_model": false, |
| "eval_frequency": 200000, |
| "eval_samples": 200000, |
| "run_final_eval": true, |
| "dead_feature_window": 1000, |
| "feature_sampling_window": 2000, |
| "n_batches_for_norm_estimate": 1000, |
| "sae_lens_version": "6.43.0", |
| "logger": { |
| "log_to_wandb": true, |
| "log_activations_store_to_wandb": false, |
| "log_optimizer_state_to_wandb": false, |
| "log_weights_to_wandb": true, |
| "wandb_project": "sae_lens_training", |
| "wandb_id": null, |
| "run_name": "synthetic-standard-16384-LR-0.0003", |
| "wandb_entity": null, |
| "wandb_log_frequency": 10, |
| "eval_every_n_wandb_logs": 100 |
| } |
| } |
