| # ============================================================================ | |
| # Daimon Training Configuration — Liberation Labs | |
| # ============================================================================ | |
| # All hyperparameters in one place. The training script reads this file. | |
| # Edit values here rather than modifying train_daimon.py directly. | |
| # | |
| # Model: Qwen3.6-35B-A3B (MoE, 35B total params, ~3B active per token) | |
| # Hardware target: 1x H200 SXM 141GB on RunPod | |
| # Method: Full-parameter SFT with DeepSpeed ZeRO-2 + Adafactor | |
| # ============================================================================ | |
| # --- Paths --- | |
| # model_id is the HuggingFace model ID or local path on the pod. | |
| # data_path points to the HF dataset repo OR a local directory with Arrow data. | |
| # output_dir MUST be on the persistent RunPod volume (/workspace/). | |
| model_id: "Qwen/Qwen3.6-35B-A3B" | |
| model_revision: "995ad96eacd98c81ed38be0c5b274b04031597b0" | |
| data_path: "HumboldtJoker/daimon-sft-data" | |
| output_dir: "/workspace/daimon-sft" | |
| # --- DeepSpeed --- | |
| # ZeRO Stage 2 with CPU-offloaded optimizer. Required for full SFT on single GPU. | |
| # Gradients are sharded across the training step; Adafactor states live in CPU RAM. | |
| deepspeed_config: "ds_config_zero2.json" | |
| # --- Sequence length --- | |
| # CRITICAL: Previous runs silently truncated 5640-token sequences to 4096. | |
| # Set this high enough for your longest training example. | |
| # The training script will pre-split any sequence exceeding this length. | |
| # Reduced from 8192 to 4096 for full SFT — activation memory scales with seq length, | |
| # and we need headroom for full gradients. Increase if your data requires it, | |
| # but monitor GPU OOM carefully. | |
| max_seq_length: 4096 | |
| # --- Optimizer --- | |
| # Adafactor: factored second moments use ~35GB CPU RAM for 35B params. | |
| # AdamW is NOT viable: fp32 momentum + variance = 35B × 4B × 2 = 280GB, | |
| # exceeding the 188GB system RAM even with full CPU offload. | |
| optimizer: "adafactor" | |
| # --- Training hyperparameters --- | |
| num_train_epochs: 1 | |
| max_steps: 10000 | |
| per_device_train_batch_size: 1 | |
| gradient_accumulation_steps: 8 | |
| # Effective batch size = per_device * 1 GPU * grad_accum = 1 * 1 * 8 = 8 | |
| # Full SFT learning rate — much lower than LoRA (2e-4) because we're updating | |
| # ALL parameters. Too high a rate destabilizes MoE routing gates. | |
| learning_rate: 5.0e-6 | |
| lr_scheduler_type: "cosine" | |
| warmup_steps: 100 | |
| # No weight decay with Adafactor — it handles regularization internally | |
| weight_decay: 0.0 | |
| max_grad_norm: 1.0 | |
| seed: 42 | |
| # --- Checkpointing --- | |
| # Full model checkpoints are ~70GB each (entire model in bf16). | |
| # save_total_limit=3 means up to ~210GB of checkpoint space needed. | |
| # With 400GB persistent volume, that leaves room for model + data + final output. | |
| save_steps: 500 | |
| save_total_limit: 3 | |
| # --- Evaluation --- | |
| eval_steps: 500 | |
| eval_strategy: "steps" | |
| # --- Logging --- | |
| logging_steps: 10 | |
| report_to: "none" | |
| # --- Precision --- | |
| bf16: true | |
| # --- Gradient checkpointing --- | |
| # Trades ~30% extra compute for roughly halved activation memory. | |
| # Critical for full SFT — without it, activations alone would be ~60GB, | |
| # leaving no room for gradients on GPU. With it, activations drop to ~20GB. | |
| gradient_checkpointing: true | |