File size: 839 Bytes
5fd477e ff79fea 5fd477e d84e4a8 ff79fea 5fd477e d84e4a8 5fd477e ff79fea 5fd477e ff79fea 5fd477e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | {
"trainer": {
"trainer_class": "TopKTrainer",
"dict_class": "AutoEncoderTopK",
"lr": 0.000282842712474619,
"steps": 61035,
"auxk_alpha": 0.03125,
"warmup_steps": 0,
"decay_start": null,
"threshold_beta": 0.999,
"threshold_start_step": 1000,
"seed": null,
"activation_dim": 6144,
"dict_size": 8192,
"k": 128,
"device": "cuda:0",
"layer": 4,
"lm_name": "blah",
"wandb_name": "AutoEncoderTopK",
"submodule_name": null
},
"buffer": {
"n_models": 12,
"d_submodule": 512,
"io": "out",
"n_ctxs": 1024,
"ctx_len": 128,
"refresh_batch_size": 512,
"out_batch_size": 8192,
"device": "cuda:0",
"rescale_acts": true
}
} |