| { |
| "model_class": "Autoregressive_Model", |
| "model_config": { |
| "name": "CrazyModel", |
| "hidden_size": 768, |
| "ffn_factor": 3.0, |
| "vocab_size": 32768, |
| "bos_token_id": 5, |
| "eos_token_id": 6, |
| "pad_token_id": 0, |
| "mask_token_id": 4, |
| "masked_substitution_rate": 0.15, |
| "num_hidden_layers": 12, |
| "num_attention_heads": 12, |
| "tie_word_embeddings": false, |
| "rms_norm_eps": 1e-06, |
| "attention_type": [], |
| "max_position_embeddings": 1024, |
| "block_size_for_attention": 128, |
| "rope_theta": 10000.0, |
| "compile_flexattn": false, |
| "bias": false, |
| "training_objective": "crazy", |
| "is_causal": true, |
| "default_layer": { |
| "attn_impl": "flash", |
| "sliding_window_size": null, |
| "positional_encoding": [ |
| "rope", |
| "alibi" |
| ], |
| "normalization": "rmsnorm", |
| "normalization_position": "pre", |
| "ffn_activation": "swiglu", |
| "hooks": {} |
| }, |
| "custom_layers": {} |
| }, |
| "training": { |
| "optimizer": "muon", |
| "lr_scheduling": true, |
| "lr": 0.0008, |
| "final_lr": 4e-05, |
| "hold_steps": 0.31, |
| "weight_decay": 0.01, |
| "scheduler": "custom", |
| "gradient_clip_val": 1.0, |
| "warmup_steps": 0.09, |
| "max_epochs": 1, |
| "accumulate_grad_batches": 5, |
| "seed": 27, |
| "save_every_n_steps": 100000, |
| "checkpoint_name": "crazy_model" |
| }, |
| "tokenizer": { |
| "type": "huggingface", |
| "pretrained_name": "mrinaldi/Gettone", |
| "varlen_strategy": "unpadding" |
| }, |
| "data": { |
| "data_root": "/home/matteo/Albertone/Albertina/micro_albertina", |
| "batch_size": 48, |
| "num_workers": 1, |
| "mdat_strategy": "Gettone_1024", |
| "mdat_view": null |
| }, |
| "save_dir": "./checkpoints_crazy", |
| "wandb_project": "CrazyModel", |
| "wandb_run_name": "CrazyModel_ALIBI" |
| } |
