| { |
| "adam_beta1": 0.9, |
| "adam_beta2": 0.999, |
| "adam_epsilon": 1e-08, |
| "beta": 0.05, |
| "dataset_manifest": "/media/nvidia/4TB2/llm-data/senryu/rl-v1/pack-v2/manifest.json", |
| "ema_inv_gamma": 1.0, |
| "ema_max_decay": 0.9999, |
| "ema_power": 0.75, |
| "enable_gradient_checkpointing": true, |
| "epochs": 1, |
| "forward_overhead_tokens": 1024, |
| "fp32_master_weights": true, |
| "gradient_checkpointing_layers": 22, |
| "gradient_clip_norm": 0.2, |
| "initial_checkpoint": "checkpoint-00012800", |
| "initial_run_dir": "/media/nvidia/4TB2/llm-runs/senryu/sft-v1", |
| "initial_weights": "ema", |
| "label_smoothing": 0.0, |
| "learning_rate": 1e-06, |
| "log_every_optimizer_steps": 1, |
| "loss_type": "apo_zero", |
| "max_length": 16512, |
| "maximum_checkpoints": 5, |
| "maximum_optimizer_steps": null, |
| "micro_batch_tokens": 32768, |
| "minimum_learning_rate_ratio": 0.1, |
| "muon_learning_rate": null, |
| "muon_lr_adjustment": "match_rms_adamw", |
| "muon_momentum": 0.95, |
| "optimizer": "adamw", |
| "output_dir": "/media/nvidia/4TB2/llm-runs/senryu/apo-v2", |
| "pairs_per_step": 32, |
| "reference_batch_tokens": 32768, |
| "save_every_optimizer_steps": 100, |
| "seed": 42, |
| "sft_weight": 0.0, |
| "sources": null, |
| "validation_pairs": 1000, |
| "warmup_ratio": 0.1, |
| "warmup_steps": null, |
| "weight_decay": 0.0 |
| } |
|
|
