# ============================================================================
# Daimon Training Configuration — Liberation Labs
# ============================================================================
# All hyperparameters in one place. The training script reads this file.
# Edit values here rather than modifying train_daimon.py directly.
#
# Model: Qwen3.6-35B-A3B (MoE, 35B total params, ~3B active per token)
# Hardware target: 1x H200 SXM 141GB on RunPod
# Method: Full-parameter SFT with DeepSpeed ZeRO-2 + Adafactor
# ============================================================================

# --- Paths ---
# model_id is the HuggingFace model ID or local path on the pod.
# data_path points to the HF dataset repo OR a local directory with Arrow data.
# output_dir MUST be on the persistent RunPod volume (/workspace/).
model_id: "Qwen/Qwen3.6-35B-A3B"
model_revision: "995ad96eacd98c81ed38be0c5b274b04031597b0"
data_path: "HumboldtJoker/daimon-sft-data"
output_dir: "/workspace/daimon-sft"

# --- DeepSpeed ---
# ZeRO Stage 2 with CPU-offloaded optimizer. Required for full SFT on single GPU.
# Gradients are sharded across the training step; Adafactor states live in CPU RAM.
deepspeed_config: "ds_config_zero2.json"

# --- Sequence length ---
# CRITICAL: Previous runs silently truncated 5640-token sequences to 4096.
# Set this high enough for your longest training example.
# The training script will pre-split any sequence exceeding this length.
# Reduced from 8192 to 4096 for full SFT — activation memory scales with seq length,
# and we need headroom for full gradients. Increase if your data requires it,
# but monitor GPU OOM carefully.
max_seq_length: 4096

# --- Optimizer ---
# Adafactor: factored second moments use ~35GB CPU RAM for 35B params.
# AdamW is NOT viable: fp32 momentum + variance = 35B × 4B × 2 = 280GB,
# exceeding the 188GB system RAM even with full CPU offload.
optimizer: "adafactor"

# --- Training hyperparameters ---
num_train_epochs: 1
max_steps: 10000
per_device_train_batch_size: 1
gradient_accumulation_steps: 8
# Effective batch size = per_device * 1 GPU * grad_accum = 1 * 1 * 8 = 8

# Full SFT learning rate — much lower than LoRA (2e-4) because we're updating
# ALL parameters. Too high a rate destabilizes MoE routing gates.
learning_rate: 5.0e-6
lr_scheduler_type: "cosine"
warmup_steps: 100
# No weight decay with Adafactor — it handles regularization internally
weight_decay: 0.0
max_grad_norm: 1.0
seed: 42

# --- Checkpointing ---
# Full model checkpoints are ~70GB each (entire model in bf16).
# save_total_limit=3 means up to ~210GB of checkpoint space needed.
# With 400GB persistent volume, that leaves room for model + data + final output.
save_steps: 500
save_total_limit: 3

# --- Evaluation ---
eval_steps: 500
eval_strategy: "steps"

# --- Logging ---
logging_steps: 10
report_to: "none"

# --- Precision ---
bf16: true

# --- Gradient checkpointing ---
# Trades ~30% extra compute for roughly halved activation memory.
# Critical for full SFT — without it, activations alone would be ~60GB,
# leaving no room for gradients on GPU. With it, activations drop to ~20GB.
gradient_checkpointing: true