| { |
| "step": 2000, |
| "tokens_consumed": 1048576000, |
| "sequences_consumed": 512000, |
| "config": { |
| "vocab_size": 129280, |
| "dim": 1536, |
| "n_layers": 16, |
| "n_heads": 12, |
| "head_dim": 128, |
| "rope_head_dim": 32, |
| "q_lora_rank": 384, |
| "o_groups": 4, |
| "o_lora_rank": 256, |
| "norm_eps": 1e-06, |
| "n_routed_experts": 12, |
| "n_shared_experts": 1, |
| "n_activated_experts": 2, |
| "moe_inter_dim": 1408, |
| "score_func": "sqrtsoftplus", |
| "route_scale": 2.5, |
| "swiglu_limit": 10.0, |
| "n_dense_layers": 2, |
| "load_balance_alpha": 0.0001, |
| "hc_mult": 3, |
| "hc_sinkhorn_iters": 8, |
| "hc_eps": 1e-06, |
| "window_size": 128, |
| "compress_ratio": 4, |
| "n_mtp_layers": 1, |
| "mtp_loss_weight": 0.3, |
| "rope_theta": 10000.0, |
| "rope_factor": 1.0, |
| "original_seq_len": 0, |
| "beta_fast": 32, |
| "beta_slow": 1, |
| "ctx_phase_boundaries": [ |
| 60000, |
| 80000, |
| 100000 |
| ], |
| "ctx_phase_lengths": [ |
| 2048, |
| 4096, |
| 8192 |
| ], |
| "yarn_scale": 40.0, |
| "yarn_alpha": 1.0, |
| "yarn_beta": 32.0, |
| "yarn_original_seq_len": 4096, |
| "micro_batch_size": 1, |
| "gradient_accumulation_steps": 32, |
| "max_lr": 0.00042, |
| "min_lr": 4.2e-05, |
| "warmup_steps": 2000, |
| "total_steps": 200000, |
| "decay_steps": 180000, |
| "weight_decay": 0.1, |
| "muon_beta": 0.95, |
| "muon_ns_steps": 5, |
| "adam_beta1": 0.9, |
| "adam_beta2": 0.95, |
| "grad_clip_norm": 1.0, |
| "init_std": 0.006, |
| "dtype": "bfloat16", |
| "fineweb_config": "default", |
| "approx_tokens_per_fineweb_doc": 800, |
| "approx_tokens_per_code_doc": 1200, |
| "approx_tokens_per_math_doc": 600, |
| "data_mix_probs": [ |
| 0.6, |
| 0.25, |
| 0.15 |
| ], |
| "save_every_steps": 400, |
| "log_every_steps": 10, |
| "push_every_steps": 400, |
| "checkpoint_dir": "/kaggle/working/zenyxv3checkpoints" |
| } |
| } |
