| { |
| "step": 91, |
| "val_bpb": 3.8118258814804027, |
| "model_config": { |
| "run": "gauss_mad_d1", |
| "device_type": "", |
| "depth": 1, |
| "aspect_ratio": 64, |
| "head_dim": 128, |
| "sequence_len": 256, |
| "vocab_size": 16, |
| "rva_blocks": [], |
| "gauss_blocks": [ |
| 0 |
| ], |
| "num_iterations": -1, |
| "target_flops": -1.0, |
| "target_param_data_ratio": 10, |
| "data_dir": "mad_data", |
| "device_batch_size": 32, |
| "total_batch_size": 8192, |
| "embedding_lr": 0.3, |
| "unembedding_lr": 0.004, |
| "grad_clip": 1.0, |
| "weight_decay": 0.0, |
| "matrix_lr": 0.02, |
| "adam_beta1": 0.8, |
| "adam_beta2": 0.95, |
| "warmup_ratio": 0.0, |
| "warmdown_ratio": 0.4, |
| "final_lr_frac": 0.0, |
| "resume_from_step": -1, |
| "eval_every": 250, |
| "eval_tokens": 65536, |
| "core_metric_every": -1, |
| "core_metric_max_per_task": 500, |
| "sample_every": -1, |
| "save_every": -1, |
| "push_checkpoints_to_hub": true, |
| "use_profiler": true, |
| "profile_step": 2, |
| "profile_micro_step": 0, |
| "memory_history_max_entries": 10000, |
| "model_tag": "gauss_mad_d1", |
| "n_layer": 1, |
| "n_head": 1, |
| "n_kv_head": 1, |
| "n_embd": 64, |
| "d_state": 32 |
| }, |
| "user_config": { |
| "run": "gauss_mad_d1", |
| "device_type": "", |
| "depth": 1, |
| "aspect_ratio": 64, |
| "head_dim": 128, |
| "sequence_len": 256, |
| "vocab_size": 16, |
| "rva_blocks": [], |
| "gauss_blocks": [ |
| 0 |
| ], |
| "num_iterations": -1, |
| "target_flops": -1.0, |
| "target_param_data_ratio": 10, |
| "data_dir": "mad_data", |
| "device_batch_size": 32, |
| "total_batch_size": 8192, |
| "embedding_lr": 0.3, |
| "unembedding_lr": 0.004, |
| "grad_clip": 1.0, |
| "weight_decay": 0.0, |
| "matrix_lr": 0.02, |
| "adam_beta1": 0.8, |
| "adam_beta2": 0.95, |
| "warmup_ratio": 0.0, |
| "warmdown_ratio": 0.4, |
| "final_lr_frac": 0.0, |
| "resume_from_step": -1, |
| "eval_every": 250, |
| "eval_tokens": 65536, |
| "core_metric_every": -1, |
| "core_metric_max_per_task": 500, |
| "sample_every": -1, |
| "save_every": -1, |
| "push_checkpoints_to_hub": true, |
| "use_profiler": true, |
| "profile_step": 2, |
| "profile_micro_step": 0, |
| "memory_history_max_entries": 10000, |
| "model_tag": "gauss_mad_d1" |
| }, |
| "device_batch_size": 32, |
| "sequence_len": 256, |
| "dataloader_state_dict": { |
| "sample_idx": 2944 |
| }, |
| "loop_state": { |
| "min_val_bpb": 3.8118258814804027, |
| "smooth_train_loss": 2.642747494703387, |
| "total_training_time": 3.2108139991760254 |
| } |
| } |
