Add training template: train_daimon_config.yaml

training-template/train_daimon_config.yaml

@@ -0,0 +1,80 @@
+# ============================================================================
+# Daimon Training Configuration — Liberation Labs
+# ============================================================================
+# All hyperparameters in one place. The training script reads this file.
+# Edit values here rather than modifying train_daimon.py directly.
+#
+# Model: Qwen3.6-35B-A3B (MoE, 35B total params, ~3B active per token)
+# Hardware target: 1x H200 SXM 141GB on RunPod
+# Method: Full-parameter SFT with DeepSpeed ZeRO-2 + Adafactor
+# ============================================================================
+
+# --- Paths ---
+# model_id is the HuggingFace model ID or local path on the pod.
+# data_path points to the HF dataset repo OR a local directory with Arrow data.
+# output_dir MUST be on the persistent RunPod volume (/workspace/).
+model_id: "Qwen/Qwen3.6-35B-A3B"
+model_revision: "995ad96eacd98c81ed38be0c5b274b04031597b0"
+data_path: "HumboldtJoker/daimon-sft-data"
+output_dir: "/workspace/daimon-sft"
+
+# --- DeepSpeed ---
+# ZeRO Stage 2 with CPU-offloaded optimizer. Required for full SFT on single GPU.
+# Gradients are sharded across the training step; Adafactor states live in CPU RAM.
+deepspeed_config: "ds_config_zero2.json"
+
+# --- Sequence length ---
+# CRITICAL: Previous runs silently truncated 5640-token sequences to 4096.
+# Set this high enough for your longest training example.
+# The training script will pre-split any sequence exceeding this length.
+# Reduced from 8192 to 4096 for full SFT — activation memory scales with seq length,
+# and we need headroom for full gradients. Increase if your data requires it,
+# but monitor GPU OOM carefully.
+max_seq_length: 4096
+
+# --- Optimizer ---
+# Adafactor: factored second moments use ~35GB CPU RAM for 35B params.
+# AdamW is NOT viable: fp32 momentum + variance = 35B × 4B × 2 = 280GB,
+# exceeding the 188GB system RAM even with full CPU offload.
+optimizer: "adafactor"
+
+# --- Training hyperparameters ---
+num_train_epochs: 1
+max_steps: 10000
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 8
+# Effective batch size = per_device * 1 GPU * grad_accum = 1 * 1 * 8 = 8
+
+# Full SFT learning rate — much lower than LoRA (2e-4) because we're updating
+# ALL parameters. Too high a rate destabilizes MoE routing gates.
+learning_rate: 5.0e-6
+lr_scheduler_type: "cosine"
+warmup_steps: 100
+# No weight decay with Adafactor — it handles regularization internally
+weight_decay: 0.0
+max_grad_norm: 1.0
+seed: 42
+
+# --- Checkpointing ---
+# Full model checkpoints are ~70GB each (entire model in bf16).
+# save_total_limit=3 means up to ~210GB of checkpoint space needed.
+# With 400GB persistent volume, that leaves room for model + data + final output.
+save_steps: 500
+save_total_limit: 3
+
+# --- Evaluation ---
+eval_steps: 500
+eval_strategy: "steps"
+
+# --- Logging ---
+logging_steps: 10
+report_to: "none"
+
+# --- Precision ---
+bf16: true
+
+# --- Gradient checkpointing ---
+# Trades ~30% extra compute for roughly halved activation memory.
+# Critical for full SFT — without it, activations alone would be ~60GB,
+# leaving no room for gradients on GPU. With it, activations drop to ~20GB.
+gradient_checkpointing: true