File size: 1,300 Bytes
2e3c313 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | {
"accelerator": "auto",
"accumulate_grad_batches": 2,
"alignment_checkpoint": "output/alignment/exp12-lr5e5-b02-b8a4-val1.3030.ckpt",
"architectures": [
"DotLMForCausalLM"
],
"auto_map": {
"AutoConfig": "modeling_dotlm.DotLMConfig",
"AutoModelForCausalLM": "modeling_dotlm.DotLMForCausalLM"
},
"batch_size": 16,
"bos_token_id": null,
"context_len": 4096,
"d_model": 768,
"dataset_path": "dataset/reasoning.json",
"devices": "auto",
"dtype": "float32",
"eos_token_id": 3,
"gradient_clip_val": 0.5,
"hidden_dim": 2048,
"initializer_range": 0.02,
"learning_rate": 5e-05,
"max_epochs": 10,
"max_seq_len": 768,
"max_steps": -1,
"model_type": "dotlm",
"n_heads": 6,
"n_kv_heads": 2,
"norm_eps": 1e-06,
"num_hidden_layers": 24,
"num_workers": 4,
"output_dir": "output/reasoning",
"pad_token_id": 0,
"precision": "bf16-mixed",
"run_prefix": "dotlm_165m",
"save_top_k": 5,
"scheduler_type": "cosine_with_warmup",
"strategy": "auto",
"theta_base": 10000.0,
"tie_word_embeddings": true,
"transformers_version": "5.5.0",
"use_cache": true,
"use_wandb": true,
"val_split": 0.01,
"val_steps": 200,
"version": "auto",
"vocab_size": 16384,
"wandb_project": "DotLM",
"warmup_steps": 50,
"weight_decay": 0.01
}
|