Add files using upload-large-folder tool

README.md

@@ -0,0 +1,12 @@
+# YuLan hybrid continued-pretraining checkpoints
+
+Archive upload is in progress. Do not assume a checkpoint is complete until `UPLOAD_COMPLETE.json` exists.
+
+- Stage15 final / Stage16a starting checkpoint: `iter_0006855`.
+- Stage16a final checkpoint: `iter_0002384`.
+- `checkpoints/`: native Megatron Core distributed checkpoints, including optimizer/training state as present in the source.
+- `hf/`: the existing Hugging Face exports recovered from the YuLan object store.
+- Training logs and launch scripts retain the stage-relative layout. HF export logs are under `hf_export_logs/`.
+- Files listed in `archive-manifest.json` are verified against remote sizes and content hashes at completion.
+
+This repository is public at the owner's request. Original training checkpoint files are read in place through task-owned hardlinks.

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/hf_export_daemon_stage16a.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+# HF auto-export daemon for stage16a (512K, CP32) — for later ckpt merging.
+# Every EXPORT_INTERVAL iterations of the training ckpt dir, produce a model-only
+# HF export and upload it to S3: hardlink-preserve the ckpt (SSD, instant),
+# convert on CPU, validate, rclone to s3://yulan/$S3_PREFIX/iter_XXXXXXX-hf/,
+# verify byte-identical, then delete local copies. Also exports the FINAL
+# iteration when training exits.
+# Zero interaction with the training process (read-only on ckpts, CPU-only conversion).
+set -uo pipefail
+
+S16A=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a
+CKPT_DIR=$S16A/checkpoints
+WORK=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work
+S3_PREFIX=f3-plain-shortconv-adamh-stage16a-cp32-512k-hf
+DONE_DIR=$WORK/done
+EXPORT_INTERVAL=80
+MIN_SSD_FREE_BYTES=350000000000   # 350G: stop exporting below this (transient work dir)
+FINAL_ITER=2384
+CONVERTER_DIR=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor
+
+mkdir -p "$WORK" "$DONE_DIR"
+log() { echo "[$(date '+%F %T')] $*"; }
+
+ckpt_complete() {
+    local d=$1
+    [ -d "$d" ] || return 1
+    [ -f "$d/.metadata" ] || return 1
+    [ "$(ls "$d" | wc -l)" -ge 131 ] || return 1
+    ! ls "$d" | grep -qiE 'temp|incomplete'
+}
+
+ssd_free_bytes() { df -B1 /mnt/ssd | tail -1 | awk '{print $4}'; }
+
+s3_bytes() {
+    source /root/debug_2440/rclone_env.sh
+    rclone size "s3:yulan/$S3_PREFIX/iter_$1-hf/" 2>/dev/null | grep -oE '[0-9]+ Byte' | grep -oE '^[0-9]+' || echo 0
+}
+
+export_one() {
+    local iter=$1
+    local tag=$(printf "%07d" "$iter")
+    local s3_dir="s3:yulan/$S3_PREFIX/iter_${tag}-hf"
+    [ -e "$DONE_DIR/${tag}.done" ] && return 0
+
+    ckpt_complete "$CKPT_DIR/iter_$tag" || { log "iter $iter: ckpt incomplete, skip"; return 1; }
+    local free=$(ssd_free_bytes)
+    [ "$free" -lt "$MIN_SSD_FREE_BYTES" ] && { log "iter $iter: SSD free ${free}B < guard, STOP exporting"; return 2; }
+
+    log "iter $iter: preserving ckpt (hardlinks)..."
+    rm -rf "$WORK/iter_${tag}_preserve" "$WORK/load_view${iter}"
+    cp -al "$CKPT_DIR/iter_$tag" "$WORK/iter_${tag}_preserve" || { log "iter $iter: preserve FAILED"; return 1; }
+    mkdir -p "$WORK/load_view${iter}"
+    ln -sfn "$WORK/iter_${tag}_preserve" "$WORK/load_view${iter}/iter_$tag"
+    echo "$iter" > "$WORK/load_view${iter}/latest_checkpointed_iteration.txt"
+
+    log "iter $iter: converting to HF (CPU)..."
+    cd "$CONVERTER_DIR"
+    env USE_CUDA=false \
+        PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud \
+        AUTO_REPAIR_GDN_CHECKPOINT=0 \
+        WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_$iter \
+        HF_SAVE_ROOT=$WORK \
+        EXTRA_ARGS="--gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo" \
+        MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml \
+        bash scripts/mcore_gdn_moe/launch-mcore2hf.sh "$WORK/load_view${iter}" --iter "$iter" \
+        > "$WORK/convert_iter${iter}.log" 2>&1
+    local rc=$?
+
+    if [ $rc -ne 0 ] || ! grep -q "Conversion completed" "$WORK/convert_iter${iter}.log"; then
+        log "iter $iter: CONVERSION FAILED (rc=$rc), keeping preserve for debug"
+        return 1
+    fi
+    # validate output
+    local hfdir="$WORK/iter_${iter}-hf"
+    local nfiles=$(find "$hfdir" -type f ! -name hf2mcore.log | wc -l)
+    local nmap=$(python3 -c "import json;print(len(json.load(open('$hfdir/model.safetensors.index.json'))['weight_map']))" 2>/dev/null || echo 0)
+    if [ "$nfiles" != "11" ] || [ "$nmap" != "32226" ]; then
+        log "iter $iter: VALIDATION FAILED (files=$nfiles weight_map=$nmap), keeping preserve"
+        return 1
+    fi
+    # upload to S3 and verify byte-identical before any cleanup
+    log "iter $iter: uploading to $s3_dir ..."
+    source /root/debug_2440/rclone_env.sh
+    rclone copy "$hfdir/" "$s3_dir/" --exclude hf2mcore.log --transfers 4 --checkers 4 \
+        --s3-chunk-size 128M --s3-upload-concurrency 8 >> "$WORK/convert_iter${iter}.log" 2>&1
+    if [ $? -ne 0 ]; then
+        log "iter $iter: S3 UPLOAD FAILED, keeping local copy at $hfdir"
+        return 1
+    fi
+    local local_bytes=$(find "$hfdir" -type f ! -name hf2mcore.log -printf "%s\n" | awk '{s+=$1} END{print s}')
+    local remote_bytes=$(s3_bytes "$tag")
+    if [ "$local_bytes" != "$remote_bytes" ]; then
+        log "iter $iter: S3 VERIFY MISMATCH local=$local_bytes remote=$remote_bytes, keeping local copy"
+        return 1
+    fi
+    rm -rf "$hfdir" "$WORK/iter_${tag}_preserve" "$WORK/load_view${iter}" /tmp/mcore2hf_daemon_$iter
+    touch "$DONE_DIR/${tag}.done"
+    log "iter $iter: exported -> $s3_dir (verified $remote_bytes bytes), local cleaned"
+    return 0
+}
+
+log "daemon start: ckpt=$CKPT_DIR interval=$EXPORT_INTERVAL s3=s3://yulan/$S3_PREFIX"
+while true; do
+    if [ -f "$CKPT_DIR/latest_checkpointed_iteration.txt" ]; then
+        iter=$(cat "$CKPT_DIR/latest_checkpointed_iteration.txt" 2>/dev/null || echo 0)
+        iter=${iter//[^0-9]/}
+        [ -z "$iter" ] && iter=0
+        target=$(( iter / EXPORT_INTERVAL * EXPORT_INTERVAL ))
+        if [ "$target" -ge "$EXPORT_INTERVAL" ]; then
+            export_one "$target"
+        fi
+        # training finished: export final iteration once, then exit
+        if ! pgrep -f 'pretrain_gp[t].py' > /dev/null; then
+            log "no training process detected; latest iter=$iter"
+            export_one "$iter" && log "final iter $iter exported; daemon exit" && exit 0
+            sleep 300  # final export failed; retry later
+        fi
+    fi
+    sleep 60
+done

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node0.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+# Stage16a (two-phase plan, run A): 40B tokens normal attention on part1_40B split.
+# 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 32 docs x 524288 = 16,777,216 tokens/step; 76288 samples = 2384 iters.
+# part1 = 39.9998B tokens (every source cut at the same doc ratio, see split_20261010/split_report.json).
+# master-port 35873. TP1 x PP1 x CP16 x EP8, cp-comm-type p2p (TE has no thd support for a2a+p2p; p2p ring attention has no head-divisibility constraint; 2xCP zigzag chunking convention unchanged). Patches: packing collator + arguments + gdn_pre_gdr whitelists allow CP16. EP8 keeps stage15 optimizer ckpt compatible. CE chunks 32.
+# After completion, stage16b resumes from this run's final ckpt with sparse attention.
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 0 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node1.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 1 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node2.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 2 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node3.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 3 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node4.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 4 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node5.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 5 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node6.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 6 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node7.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 7 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node0.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+# Stage16a (two-phase plan, run A): 40B tokens normal attention on part1_40B split.
+# 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 32 docs x 524288 = 16,777,216 tokens/step; 76288 samples = 2384 iters.
+# part1 = 39.9998B tokens (every source cut at the same doc ratio, see split_20261010/split_report.json).
+# BACKUP PLAN resume: CP16+EP16, loads the fully-reshardable ckpt produced by stage16a_reshard_fr_20261011a (runbook in header of reshard_node0.sh).
+# After completion, stage16b resumes from this run's final ckpt with sparse attention.
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 0 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node1.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 1 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node2.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 2 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node3.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 3 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node4.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 4 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node5.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 5 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node6.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 6 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node7.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 7 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node0.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+# Stage16a (two-phase plan, run A): 40B tokens normal attention on part1_40B split.
+# 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 32 docs x 524288 = 16,777,216 tokens/step; 76288 samples = 2384 iters.
+# part1 = 39.9998B tokens (every source cut at the same doc ratio, see split_20261010/split_report.json).
+# RESUME from iter_0000040 with CP32 (16K tokens/rank = stage15-proven memory profile). continual-pretrain DROPPED so iteration/optimizer/scheduler/RNG resume from the ckpt.
+# After completion, stage16b resumes from this run's final ckpt with sparse attention.
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 0 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node1.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 1 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node2.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 2 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node3.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 3 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node4.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 4 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node5.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 5 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node6.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 6 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node7.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0).
+# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling).
+# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step.
+# TODO before launch: fill in --data-path (final link dir), --data-cache-path,
+#   --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32).
+# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values.
+# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 7 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit