Add files using upload-large-folder tool

.gitattributes

@@ -41,3 +41,7 @@ f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_2
 f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_021843-lzgz3u05/run-lzgz3u05.wandb filter=lfs diff=lfs merge=lfs -text
 f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261010_231200-gdb9n0eb/run-gdb9n0eb.wandb filter=lfs diff=lfs merge=lfs -text
 f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_160551-acr8mx9p/run-acr8mx9p.wandb filter=lfs diff=lfs merge=lfs -text
+f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__18_0.distcp filter=lfs diff=lfs merge=lfs -text
+f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__29_1.distcp filter=lfs diff=lfs merge=lfs -text
+f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__21_0.distcp filter=lfs diff=lfs merge=lfs -text
+f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__49_0.distcp filter=lfs diff=lfs merge=lfs -text

f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node0.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0).
+# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B).
+# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export GDN_LR_MULT_ALPHA=1.0
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 0 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node1.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0).
+# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B).
+# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export GDN_LR_MULT_ALPHA=1.0
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 1 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node2.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0).
+# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B).
+# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export GDN_LR_MULT_ALPHA=1.0
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 2 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node3.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0).
+# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B).
+# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export GDN_LR_MULT_ALPHA=1.0
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 3 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node4.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0).
+# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B).
+# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export GDN_LR_MULT_ALPHA=1.0
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 4 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node5.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0).
+# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B).
+# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export GDN_LR_MULT_ALPHA=1.0
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 5 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node6.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0).
+# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B).
+# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export GDN_LR_MULT_ALPHA=1.0
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 6 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node7.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0).
+# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B).
+# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871.
+ulimit -n 1048576
+# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json
+# (config environment applied first; explicit exports below win via setdefault semantics).
+# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP
+# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP).
+export MOE_ROUTER_SQRT_GATE=1
+export MOE_ROUTER_SCORE_FUNCTION=softmax
+export TORCH_NCCL_TRACE_BUFFER_SIZE=0
+export MOE_AMPERE_GROUPED_MLP=1
+export NCCL_IB_PCI_RELAXED_ORDERING=1
+export NCCL_MIN_NCHANNELS=8
+export GDN_ALPHA_WEIGHT_SCALE=1.0
+export NCCL_RAS_ENABLE=0
+export TORCH_NCCL_TRACE_CPP_STACK=0
+export NCCL_SOCKET_IFNAME=eth0
+export GDN_EPS=1e-6
+export NCCL_NET_GDR_LEVEL=3
+export GDN_PRE_GDR_FUSION=0
+export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512
+export NCCL_DEBUG=WARN
+export MOE_TOKEN_DISPATCHER_TYPE=alltoall
+export NCCL_TIMEOUT=18000
+export NCCL_IB_HCA=mlx5
+export GDN_CP_DISABLE_LAYOUT_BARRIER=0
+export NCCL_IB_GID_INDEX=3
+export TORCH_PROFILER_WITH_STACK=1
+export MOE_TRITON_SHARED_EXPERT_OVERLAP=0
+export MOE_EXPERT_CAPACITY_FACTOR=0
+export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true
+export GDN_ALPHA_PROFILE_NAME=a05_lr
+export NCCL_IB_QPS_PER_CONNECTION=4
+export GDN_PRE_GDR_FUSION_OVERRIDE=0
+export TORCH_NCCL_DUMP_ON_TIMEOUT=0
+export MOE_FFN_HIDDEN_SIZE=512
+export MOE_ROUTER_NORM_TOPK_PROB=true
+export FLA_GDN_CP_STATE_PRECISION=bf16
+export NCCL_IB_TIMEOUT=23
+export MOE_ROUTER_TOPK=4
+export MOE_AUX_LOSS_COEFF=0.05
+export GDN_SPLIT_IN_PROJ=1
+export TORCH_FR_CPP_STACK=0
+export OMP_NUM_THREADS=4
+export TORCH_FR_BUFFER_SIZE=0
+export FLA_GDN_NUMERICAL_CP_SIZE=0
+export GDN_FUSED_SPLIT_FORWARD=1
+export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix
+export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0
+export MOE_LAYER_FREQ=([0]*2+[1]*54)
+export NCCL_CROSS_NIC=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe
+export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention
+export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed
+export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton
+export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions
+export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+export GDN_LR_MULT_ALPHA=1.0
+export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true
+exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 7 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit

f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/config.yaml

@@ -0,0 +1,2177 @@
+_wandb:
+    value:
+        cli_version: 0.23.0
+        e:
+            i4bo5dz9n6n9rgehz11zfcuyno7peo6e:
+                args:
+                    - --use-mcore-models
+                    - --hybrid-attention-ratio
+                    - "0.0625"
+                    - --hybrid-mlp-ratio
+                    - "0.5"
+                    - --is-hybrid-model
+                    - --linear-attention-type
+                    - gated_delta_net
+                    - --linear-conv-kernel-dim
+                    - "4"
+                    - --linear-key-head-dim
+                    - "64"
+                    - --linear-value-head-dim
+                    - "64"
+                    - --linear-num-key-heads
+                    - "8"
+                    - --linear-num-value-heads
+                    - "32"
+                    - --disable-bias-linear
+                    - --seq-length
+                    - "131072"
+                    - --max-position-embeddings
+                    - "131072"
+                    - --num-layers
+                    - "56"
+                    - --hidden-size
+                    - "2048"
+                    - --ffn-hidden-size
+                    - "4800"
+                    - --num-attention-heads
+                    - "32"
+                    - --init-method-std
+                    - "0.02"
+                    - --attention-dropout
+                    - "0.0"
+                    - --hidden-dropout
+                    - "0.0"
+                    - --normalization
+                    - RMSNorm
+                    - --swiglu
+                    - --group-query-attention
+                    - --num-query-groups
+                    - "8"
+                    - --no-masked-softmax-fusion
+                    - --use-flash-attn
+                    - --add-qkv-bias
+                    - --mtp-num-layers
+                    - "0"
+                    - --mtp-loss-scaling-factor
+                    - "0.1"
+                    - --norm-epsilon
+                    - "1e-06"
+                    - --mtp-linear-attention-type
+                    - gated_delta_net
+                    - --position-embedding-type
+                    - none
+                    - --linear-attention-freq
+                    - '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]'
+                    - --optimizer
+                    - adam
+                    - --adam-beta1
+                    - "0.9"
+                    - --adam-beta2
+                    - "0.95"
+                    - --adam-eps
+                    - "1e-15"
+                    - --weight-decay
+                    - "0.1"
+                    - --clip-grad
+                    - "1.0"
+                    - --data-path
+                    - /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp
+                    - --split
+                    - 999,1,0
+                    - --data-cache-path
+                    - /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2
+                    - --no-create-attention-mask-in-dataloader
+                    - --num-experts
+                    - "192"
+                    - --moe-grouped-gemm
+                    - --moe-router-topk
+                    - "4"
+                    - --moe-router-score-function
+                    - softmax
+                    - --overlap-param-gather
+                    - --overlap-grad-reduce
+                    - --moe-expert-capacity-factor
+                    - "0"
+                    - --moe-shared-expert-intermediate-size
+                    - "512"
+                    - --moe-router-dtype
+                    - fp32
+                    - --moe-permute-fusion
+                    - --moe-ffn-hidden-size
+                    - "512"
+                    - --moe-shared-expert-gate
+                    - --moe-shared-expert-overlap
+                    - --moe-layer-freq
+                    - ([0]*2+[1]*54)
+                    - --moe-aux-loss-coeff
+                    - "0.05"
+                    - --moe-router-load-balancing-type
+                    - global_aux_loss
+                    - --micro-batch-size
+                    - "1"
+                    - --global-batch-size
+                    - "128"
+                    - --lr
+                    - "5e-4"
+                    - --train-samples
+                    - "877440"
+                    - --lr-warmup-samples
+                    - "0"
+                    - --lr-decay-samples
+                    - "877440"
+                    - --lr-decay-style
+                    - constant
+                    - --min-lr
+                    - "0.0005"
+                    - --bf16
+                    - --async-save
+                    - --ckpt-fully-parallel-load
+                    - --auto-detect-ckpt-format
+                    - --load-complemental-dataset
+                    - loss_mask
+                    - --tensor-model-parallel-size
+                    - "1"
+                    - --pipeline-model-parallel-size
+                    - "1"
+                    - --context-parallel-size
+                    - "8"
+                    - --expert-model-parallel-size
+                    - "8"
+                    - --sequence-parallel
+                    - --use-distributed-optimizer
+                    - --log-interval
+                    - "1"
+                    - --log-throughput
+                    - --save-interval
+                    - "40"
+                    - --eval-interval
+                    - "99999999"
+                    - --eval-iters
+                    - "10"
+                    - --increase-log-level-iters
+                    - "5"
+                    - --timing-log-level
+                    - "0"
+                    - --wandb-project
+                    - qwen2.5_3b_mom_cpt
+                    - --tokenizer-type
+                    - HuggingFaceTokenizer
+                    - --tokenizer-model
+                    - /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp
+                    - --no-adam-bias-correction
+                    - --corrected-weight-decay
+                    - --auto-generate-cu-seqlens
+                    - --cp-comm-type
+                    - a2a
+                    - --log-moe-activations
+                    - --final-logits-clamp-value
+                    - "30"
+                    - --attention-output-gate
+                    - --gated-norm
+                    - --gated-norm-lora-rank
+                    - "16"
+                    - --dataloader-type
+                    - cyclic
+                    - --lm-loss-normalization
+                    - global-token
+                    - --prefetch-factor
+                    - "8"
+                    - --activation-func-clamp-value
+                    - "7"
+                    - --activation-func-clamp-mode
+                    - soft
+                    - --gdn-split-in-proj
+                    - --gdn-fused-split-forward
+                    - --gdn-alpha-weight-scale
+                    - "1.0"
+                    - --gdn-alpha-weight-scale-mode
+                    - matrix
+                    - --gdn-lr-mult-alpha
+                    - "0.5"
+                    - --start-samples
+                    - "0"
+                    - --ckpt-format
+                    - torch_dist
+                    - --rerun-mode
+                    - validate_results
+                    - --distributed-timeout-minutes
+                    - "120"
+                    - --dist-ckpt-strictness
+                    - assume_ok_unexpected
+                    - --gdn-backend
+                    - fla
+                    - --gdn-gated-norm-backend
+                    - linghe
+                    - --moe-router-gemm-backend
+                    - te
+                    - --log-params-norm
+                    - --timing-log-option
+                    - all
+                    - --no-barrier-with-level-1-timing
+                    - --log-metrics-interval
+                    - "10"
+                    - --cross-entropy-loss-fusion
+                    - --cross-entropy-fusion-impl
+                    - linear_triton
+                    - --linear-cross-entropy-num-chunks
+                    - "8"
+                    - --num-workers
+                    - "4"
+                    - --moe-router-sqrt-gate
+                    - --moe-router-fusion
+                    - --moe-token-dispatcher-type
+                    - alltoall
+                    - --log-hidden-states
+                    - embeddings
+                    - residual_attn
+                    - input_layernorm
+                    - attention::q_layernorm
+                    - attention::k_layernorm
+                    - attention::core_attention
+                    - attention::o_proj
+                    - residual_mlp
+                    - pre_mlp_layernorm
+                    - mlp
+                    - moe::router
+                    - moe::shared_experts
+                    - moe::routed_experts
+                    - gated_delta::in_proj
+                    - gated_delta::conv1d
+                    - gated_delta::query
+                    - gated_delta::key
+                    - gated_delta::value
+                    - gated_delta::gate
+                    - gated_delta::beta
+                    - gated_delta::alpha
+                    - gated_delta::g
+                    - gated_delta::core_attention
+                    - gated_delta::out_norm
+                    - gated_delta::out_proj
+                    - final_layernorm
+                    - --log-params
+                    - attention::linear_qkv
+                    - attention::o_proj
+                    - mlp::linear_fc1
+                    - mlp::linear_fc2
+                    - input_layernorm
+                    - pre_mlp_layernorm
+                    - embedding
+                    - lm_head
+                    - gated_delta::in_proj
+                    - gated_delta::in_proj_bias
+                    - gated_delta::conv1d
+                    - gated_delta::conv1d_bias
+                    - gated_delta::dt_bias
+                    - gated_delta::A_log
+                    - gated_delta::out_norm
+                    - gated_delta::out_norm_bias
+                    - gated_delta::out_proj
+                    - gated_delta::out_proj_bias
+                    - --log-update-rms
+                    - --log-per-module-grad-rms
+                    - --log-per-layer-grad-rms
+                    - --document-packing-algorithm
+                    - obfd
+                    - --exit-on-missing-checkpoint
+                    - --no-initialization
+                    - --untie-embeddings-and-output-weights
+                    - --recompute-granularity
+                    - full
+                    - --recompute-method
+                    - block
+                    - --recompute-num-layers
+                    - "56"
+                    - --no-save-step-one
+                    - --save-retain-interval
+                    - "1000000000"
+                    - --exit-signal-handler
+                    - --fast-thd-a2a
+                    - --hybrid-adamh
+                    - --reuse-training-communicators
+                    - --ddp-bucket-size
+                    - "40000000"
+                    - --load
+                    - /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints
+                    - --continual-pretrain
+                    - --save
+                    - /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints
+                    - --tensorboard-dir
+                    - /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard
+                    - --wandb-save-dir
+                    - /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb
+                    - --wandb-exp-name
+                    - stage14_120B_8node_gdnlralpha_off_20260925a
+                    - --runtime-audit-dir
+                    - /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit
+                cpu_count: 64
+                cpu_count_logical: 128
+                cudaVersion: "13.3"
+                disk:
+                    /:
+                        total: "31526391808"
+                        used: "1319878656"
+                email: huyiwen0301@hotmail.com
+                executable: /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python
+                gpu: NVIDIA A800-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-40999d1b-2b80-14f4-48d7-334e7dc3e650
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-c854c8e4-7352-b317-f9e8-7ab1d7a12c08
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-f7d2842f-48e9-649c-55a4-4efe7a6e20f7
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-03925119-14d4-a11c-d740-0ba854285052
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-7cbabdfb-9388-3fbd-babc-d0b03ac9f1bb
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-4999b9f4-9851-ce6a-4296-c230e2813b5f
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-9e23b382-2f8e-e424-f025-c2b311d96c54
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-4241d9ca-7546-c1c3-dba8-bdc5786a3777
+                host: cs-3554a-53569-server
+                memory:
+                    total: "1081804955648"
+                os: Linux-5.15.0-94-generic-x86_64-with-glibc2.39
+                program: /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py
+                python: CPython 3.10.19
+                root: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb
+                startedAt: "2026-10-03T07:36:55.078128Z"
+                writerId: i4bo5dz9n6n9rgehz11zfcuyno7peo6e
+        m: []
+        python_version: 3.10.19
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+            "3":
+                - 2
+                - 13
+                - 16
+                - 61
+            "4": 3.10.19
+            "5": 0.23.0
+            "6": 5.13.0
+            "12": 0.23.0
+            "13": linux-x86_64
+account_for_embedding_in_pipeline_split:
+    value: false
+account_for_loss_in_pipeline_split:
+    value: false
+accumulate_allreduce_grads_in_fp32:
+    value: true
+activation_func_clamp_mode:
+    value: soft
+activation_func_clamp_value:
+    value: 7
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.95
+adam_bias_correction:
+    value: false
+adam_eps:
+    value: 1e-15
+adamh_eps:
+    value: 1e-08
+adamh_radius:
+    value: null
+adamh_scope:
+    value: lm_head
+adamw_lr_mup_scaler:
+    value: false
+add_bias_linear:
+    value: false
+add_position_embedding:
+    value: true
+add_qkv_bias:
+    value: true
+adlr_autoresume:
+    value: false
+adlr_autoresume_interval:
+    value: 1000
+align_grad_reduce:
+    value: true
+align_param_gather:
+    value: false
+allow_ambiguous_pad_tokens:
+    value: false
+app_tag_run_name:
+    value: null
+app_tag_run_version:
+    value: 0.0.0
+apply_layernorm_1p:
+    value: false
+apply_query_key_layer_scaling:
+    value: false
+apply_residual_connection_post_layernorm:
+    value: false
+apply_rope_fusion:
+    value: false
+async_save:
+    value: true
+async_tensor_model_parallel_allreduce:
+    value: true
+attention_backend:
+    value: auto
+attention_dropout:
+    value: 0
+attention_output_gate:
+    value: true
+attention_output_gate_upcycling:
+    value: false
+attention_softmax_in_fp32:
+    value: false
+attn_k_token_shift:
+    value: null
+attn_output_gate:
+    value: null
+attn_q_token_shift:
+    value: null
+attn_token_shift:
+    value: null
+attn_v_token_shift:
+    value: null
+auto_detect_ckpt_format:
+    value: true
+auto_generate_cu_seqlens:
+    value: true
+barrier_with_L1_time:
+    value: false
+benchmark_eval:
+    value: false
+benchmark_global_batch:
+    value: null
+benchmark_interval:
+    value: null
+benchmark_micro_batch:
+    value: null
+benchmark_sequence_length:
+    value: null
+benchmark_tasks:
+    value: null
+bert_binary_head:
+    value: true
+bert_embedder_type:
+    value: megatron
+bert_load:
+    value: null
+bf16:
+    value: true
+bias_dropout_fusion:
+    value: true
+bias_gelu_fusion:
+    value: false
+bias_swiglu_fusion:
+    value: true
+biencoder_projection_dim:
+    value: 0
+biencoder_shared_query_context_model:
+    value: false
+block_data_path:
+    value: null
+cache_mla_latents:
+    value: false
+calc_ft_timeouts:
+    value: false
+calculate_per_token_loss:
+    value: false
+check_for_large_grads:
+    value: false
+check_for_nan_in_loss_and_grad:
+    value: true
+check_for_spiky_loss:
+    value: false
+check_weight_hash_across_dp_replicas_interval:
+    value: null
+ckpt_assume_constant_structure:
+    value: false
+ckpt_convert_format:
+    value: null
+ckpt_convert_save:
+    value: null
+ckpt_convert_update_legacy_dist_opt_format:
+    value: false
+ckpt_format:
+    value: torch_dist
+ckpt_fully_parallel_load:
+    value: true
+ckpt_fully_parallel_save:
+    value: true
+ckpt_fully_parallel_save_deprecated:
+    value: false
+ckpt_step:
+    value: null
+classes_fraction:
+    value: 1
+clip_grad:
+    value: 1
+clone_scatter_output_in_embedding:
+    value: true
+config_logger_dir:
+    value: ""
+consumed_train_samples:
+    value: 0
+consumed_valid_samples:
+    value: 0
+context_parallel_size:
+    value: 8
+continual_pretrain:
+    value: true
+continual_pretrain_preserve_opt_param_scheduler:
+    value: false
+corrected_weight_decay:
+    value: true
+cp_comm_type:
+    value:
+        - a2a
+create_attention_mask_in_dataloader:
+    value: false
+cross_entropy_fusion_impl:
+    value: linear_triton
+cross_entropy_loss_fusion:
+    value: true
+cuda_graph_impl:
+    value: none
+cuda_graph_scope:
+    value: []
+cuda_graph_warmup_steps:
+    value: 3
+data_args_path:
+    value: null
+data_cache_path:
+    value: /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2
+data_parallel_random_init:
+    value: false
+data_parallel_sharding_strategy:
+    value: no_shard
+data_parallel_size:
+    value: 8
+data_path:
+    value:
+        - /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp
+data_per_class_fraction:
+    value: 1
+data_sharding:
+    value: true
+dataloader_defer_npy_index_mmap:
+    value: false
+dataloader_fast_cache_load:
+    value: false
+dataloader_type:
+    value: cyclic
+dc_shortconv:
+    value: false
+dc_shortconv_optimizer_state:
+    value: conservative
+dc_shortconv_upcycling:
+    value: false
+ddp_average_in_collective:
+    value: false
+ddp_bucket_size:
+    value: 40000000
+ddp_num_buckets:
+    value: null
+ddp_pad_buckets_for_high_nccl_busbw:
+    value: false
+decode_only_cuda_graphs:
+    value: false
+decoder_first_pipeline_num_layers:
+    value: null
+decoder_last_pipeline_num_layers:
+    value: null
+decoder_num_layers:
+    value: null
+decoder_seq_length:
+    value: null
+decoupled_lr:
+    value: null
+decoupled_min_lr:
+    value: null
+decrease_batch_size_if_needed:
+    value: false
+defer_embedding_wgrad_compute:
+    value: false
+defer_grad_checks:
+    value: false
+delay_wgrad_compute:
+    value: false
+deprecated_use_mcore_models:
+    value: true
+deterministic_mode:
+    value: false
+dino_bottleneck_size:
+    value: 256
+dino_freeze_last_layer:
+    value: 1
+dino_head_hidden_size:
+    value: 2048
+dino_local_crops_number:
+    value: 10
+dino_local_img_size:
+    value: 96
+dino_norm_last_layer:
+    value: false
+dino_teacher_temp:
+    value: 0.07
+dino_warmup_teacher_temp:
+    value: 0.04
+dino_warmup_teacher_temp_epochs:
+    value: 30
+disable_bf16_reduced_precision_matmul:
+    value: false
+disable_chunked_prefill:
+    value: false
+disable_explicit_attention_mask:
+    value: false
+disable_mamba_mem_eff_path:
+    value: false
+disable_straggler_on_startup:
+    value: false
+disable_symmetric_registration:
+    value: false
+dist_ckpt_format_deprecated:
+    value: null
+dist_ckpt_optim_fully_reshardable:
+    value: false
+dist_ckpt_save_pre_mcore_014:
+    value: false
+dist_ckpt_strictness:
+    value: assume_ok_unexpected
+distrib_optim_fully_reshardable_mem_efficient:
+    value: false
+distribute_saved_activations:
+    value: false
+distributed_backend:
+    value: nccl
+distributed_timeout_minutes:
+    value: 120
+distributed_timeout_seconds_after_init:
+    value: null
+document_packing_algorithm:
+    value: obfd
+document_packing_dp_size:
+    value: null
+dump_param_to_param_group_map:
+    value: null
+dynamic_context_parallel:
+    value: false
+dynamic_context_parallel_sizes:
+    value: null
+dynamic_cp_candidate_sizes:
+    value: null
+dynamic_cp_gdn_policy:
+    value: null
+dynamic_cp_moe_policy:
+    value: null
+dynamic_cp_on_error:
+    value: error
+dynamic_cp_scheduler:
+    value: default_dynamic_cp
+dynamic_cp_wave_coexecution:
+    value: false
+dynamic_cp_workspace_max_tokens:
+    value: null
+emb_deviation_loss_coeff:
+    value: 0
+emb_deviation_type:
+    value: null
+embedding_init_method_std:
+    value: null
+embedding_path:
+    value: null
+empty_unused_memory_level:
+    value: 0
+enable_cuda_graph:
+    value: false
+enable_debug_logging:
+    value: false
+enable_experimental:
+    value: false
+enable_ft_package:
+    value: false
+enable_full_sharding_in_hsdp:
+    value: false
+enable_gloo_process_groups:
+    value: true
+enable_msc:
+    value: true
+enable_one_logger:
+    value: true
+encoder_num_layers:
+    value: 56
+encoder_seq_length:
+    value: 131072
+end_weight_decay:
+    value: 0.1
+eod_mask_loss:
+    value: false
+error_injection_rate:
+    value: 0
+error_injection_type:
+    value: transient_error
+eval_interval:
+    value: 99999999
+eval_iters:
+    value: 10
+evidence_data_path:
+    value: null
+exit_duration_in_mins:
+    value: null
+exit_interval:
+    value: null
+exit_on_missing_checkpoint:
+    value: true
+exit_signal_handler:
+    value: true
+exp_avg_dtype:
+    value: torch.float32
+exp_avg_sq_dtype:
+    value: torch.float32
+expert_model_parallel_size:
+    value: 8
+expert_tensor_parallel_size:
+    value: 1
+export_force_local_attention:
+    value: false
+export_kd_cfg:
+    value: null
+export_kd_teacher_ckpt_format:
+    value: null
+export_kd_teacher_load:
+    value: null
+export_kv_cache_quant:
+    value: false
+export_legacy_megatron:
+    value: false
+export_model_type:
+    value: GPTModel
+export_moe_apply_probs_on_input:
+    value: false
+export_offline_model:
+    value: false
+export_qk_l2_norm:
+    value: false
+export_quant_cfg:
+    value: null
+export_real_quant_cfg:
+    value: None
+export_te_mcore_model:
+    value: false
+external_cuda_graph:
+    value: false
+fast_thd_a2a:
+    value: true
+ffn_hidden_size:
+    value: 4800
+ffn_intermediate_token_shift:
+    value: null
+ffn_token_shift:
+    value: null
+final_logits_clamp_value:
+    value: 30
+fine_grained_activation_offloading:
+    value: false
+finetune:
+    value: false
+finetune_data_split:
+    value: train
+finetune_hf_dataset:
+    value: null
+first_last_layers_bf16:
+    value: false
+flash_decode:
+    value: false
+fp4:
+    value: null
+fp4_param:
+    value: false
+fp4_recipe:
+    value: nvfp4
+fp8:
+    value: null
+fp8_amax_compute_algo:
+    value: most_recent
+fp8_amax_history_len:
+    value: 1
+fp8_interval:
+    value: 1
+fp8_margin:
+    value: 0
+fp8_param_gather:
+    value: false
+fp8_recipe:
+    value: delayed
+fp8_wgrad:
+    value: true
+fp16:
+    value: false
+fp16_lm_cross_entropy:
+    value: false
+fp32_residual_connection:
+    value: false
+freeze_layernorm_weight:
+    value: false
+freeze_non_mamba:
+    value: false
+fsdp_double_buffer:
+    value: false
+full_validation:
+    value: false
+gated_norm:
+    value: true
+gated_norm_lora_rank:
+    value: 16
+gated_norm_upcycling:
+    value: false
+gdn_alpha_weight_scale:
+    value: 1
+gdn_alpha_weight_scale_mode:
+    value: matrix
+gdn_backend:
+    value: fla
+gdn_fused_split_forward:
+    value: true
+gdn_gated_norm_backend:
+    value: linghe
+gdn_lr_mult_alpha:
+    value: 0.5
+gdn_lr_mult_beta:
+    value: 1
+gdn_lr_mult_k:
+    value: 1
+gdn_lr_mult_q:
+    value: 1
+gdn_lr_mult_v:
+    value: 1
+gdn_postinit_alog_shift:
+    value: 0
+gdn_postinit_alpha_scale:
+    value: 1
+gdn_postinit_key_scale:
+    value: 1
+gdn_pre_gdr_fusion:
+    value: false
+gdn_pre_gdr_fusion_deterministic_weight_gradient:
+    value: false
+gdn_pre_gdr_fusion_high_precision_backward:
+    value: false
+gdn_shortconv_separate_gate:
+    value: false
+gdn_shortconv_separate_gate_upcycling:
+    value: false
+gdn_split_in_proj:
+    value: true
+geglu:
+    value: false
+global_batch_size:
+    value: 128
+glu_linear_offset:
+    value: 0
+grad_reduce_in_bf16:
+    value: false
+gradient_accumulation_fusion:
+    value: true
+gradient_reduce_div_fusion:
+    value: true
+group_query_attention:
+    value: true
+grpo_clamp_eps_lower:
+    value: 0.01
+grpo_clamp_eps_upper:
+    value: 0.01
+grpo_default_temperature:
+    value: 1
+grpo_default_top_p:
+    value: 0
+grpo_entropy_term_weight:
+    value: 0
+grpo_filter_groups_with_same_reward:
+    value: false
+grpo_group_size:
+    value: 2
+grpo_iterations:
+    value: 2
+grpo_kl_beta:
+    value: 0.001
+grpo_prompts_per_step:
+    value: 32
+head_lr_mult:
+    value: 1
+heterogeneous_layers_config_encoded_json:
+    value: null
+heterogeneous_layers_config_path:
+    value: null
+hidden_dropout:
+    value: 0
+hidden_size:
+    value: 2048
+hidden_states_shrink:
+    value: 1
+hierarchical_context_parallel_sizes:
+    value: null
+high_priority_stream_groups:
+    value: []
+hybrid_adamh:
+    value: true
+hybrid_attention_ratio:
+    value: 0.0625
+hybrid_context_parallel:
+    value: false
+hybrid_mlp_ratio:
+    value: 0.5
+hybrid_override_pattern:
+    value: null
+hysteresis:
+    value: 2
+ict_head_size:
+    value: null
+ict_load:
+    value: null
+img_h:
+    value: 224
+img_w:
+    value: 224
+increase_log_level_interval:
+    value: 1000
+increase_log_level_iters:
+    value: 5
+indexer_batch_size:
+    value: 128
+indexer_log_interval:
+    value: 1000
+inference_batch_times_seqlen_threshold:
+    value: -1
+inference_dynamic_batching:
+    value: false
+inference_dynamic_batching_block_size:
+    value: 256
+inference_dynamic_batching_buffer_guaranteed_fraction:
+    value: 0.2
+inference_dynamic_batching_buffer_overflow_factor:
+    value: null
+inference_dynamic_batching_buffer_size_gb:
+    value: 40
+inference_dynamic_batching_max_requests_override:
+    value: null
+inference_dynamic_batching_max_tokens_override:
+    value: null
+inference_dynamic_batching_num_cuda_graphs:
+    value: 16
+inference_dynamic_batching_track_paused_request_events:
+    value: false
+inference_dynamic_batching_unified_memory_level:
+    value: 0
+inference_max_batch_size:
+    value: 8
+inference_max_seq_length:
+    value: 2560
+inference_rng_tracker:
+    value: false
+init_method_std:
+    value: 0.02
+init_method_xavier_uniform:
+    value: false
+init_model_with_meta_device:
+    value: false
+initial_loss_scale:
+    value: 4294967296
+inprocess_active_world_size:
+    value: 64
+inprocess_barrier_timeout:
+    value: 120
+inprocess_completion_timeout:
+    value: 120
+inprocess_empty_cuda_cache:
+    value: false
+inprocess_granularity:
+    value: node
+inprocess_hard_timeout:
+    value: 90
+inprocess_heartbeat_interval:
+    value: 30
+inprocess_heartbeat_timeout:
+    value: 60
+inprocess_last_call_wait:
+    value: 1
+inprocess_max_iterations:
+    value: null
+inprocess_monitor_process_interval:
+    value: 1
+inprocess_monitor_thread_interval:
+    value: 1
+inprocess_progress_watchdog_interval:
+    value: 1
+inprocess_restart:
+    value: false
+inprocess_soft_timeout:
+    value: 60
+inprocess_termination_grace_time:
+    value: 1
+is_hybrid_model:
+    value: true
+iter_per_epoch:
+    value: 1250
+iterations_to_skip:
+    value: []
+keep_fp8_transpose_cache:
+    value: false
+kitchen_config_file:
+    value: null
+kitchen_recipe_number:
+    value: null
+kv_channels:
+    value: 64
+kv_lora_rank:
+    value: 32
+langrl_env_config:
+    value: null
+langrl_external_server:
+    value: false
+langrl_inference_server_conversation_template:
+    value: null
+langrl_inference_server_type:
+    value: inplace_megatron
+lazy_mpu_init:
+    value: null
+legacy_tokenizer:
+    value: false
+linear_attention_freq:
+    value:
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 0
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 0
+        - 0
+        - 1
+        - 0
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 0
+        - 1
+        - 0
+        - 0
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+linear_attention_type:
+    value: gated_delta_net
+linear_conv_kernel_dim:
+    value: 4
+linear_cross_entropy_accumulate_grad_in_fp32:
+    value: true
+linear_cross_entropy_num_chunks:
+    value: 8
+linear_key_head_dim:
+    value: 64
+linear_num_key_heads:
+    value: 8
+linear_num_value_heads:
+    value: 32
+linear_rope_mode:
+    value: none
+linear_value_head_dim:
+    value: 64
+lm_head_alpha:
+    value: 1
+lm_loss_normalization:
+    value: global-token
+load:
+    value: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints
+load_complemental_dataset:
+    value:
+        - loss_mask
+load_main_params_from_ckpt:
+    value: null
+local_rank:
+    value: 7
+log_attn_logits:
+    value: false
+log_energy:
+    value: false
+log_hidden_states:
+    value:
+        - embeddings
+        - residual_attn
+        - input_layernorm
+        - attention::q_layernorm
+        - attention::k_layernorm
+        - attention::core_attention
+        - attention::o_proj
+        - residual_mlp
+        - pre_mlp_layernorm
+        - mlp
+        - moe::router
+        - moe::shared_experts
+        - moe::routed_experts
+        - gated_delta::in_proj
+        - gated_delta::conv1d
+        - gated_delta::query
+        - gated_delta::key
+        - gated_delta::value
+        - gated_delta::gate
+        - gated_delta::beta
+        - gated_delta::alpha
+        - gated_delta::g
+        - gated_delta::core_attention
+        - gated_delta::out_norm
+        - gated_delta::out_proj
+        - final_layernorm
+log_interval:
+    value: 1
+log_logits_z_loss:
+    value: false
+log_loss_scale_to_tensorboard:
+    value: true
+log_memory_to_tensorboard:
+    value: false
+log_metrics_interval:
+    value: 10
+log_moe_activations:
+    value: true
+log_moe_params:
+    value: false
+log_moe_router_metrics:
+    value: false
+log_num_zeros_in_grad:
+    value: false
+log_padding_stats:
+    value: false
+log_params:
+    value:
+        - attention::linear_qkv
+        - attention::o_proj
+        - mlp::linear_fc1
+        - mlp::linear_fc2
+        - input_layernorm
+        - pre_mlp_layernorm
+        - embedding
+        - lm_head
+        - gated_delta::in_proj
+        - gated_delta::in_proj_bias
+        - gated_delta::conv1d
+        - gated_delta::conv1d_bias
+        - gated_delta::dt_bias
+        - gated_delta::A_log
+        - gated_delta::out_norm
+        - gated_delta::out_norm_bias
+        - gated_delta::out_proj
+        - gated_delta::out_proj_bias
+log_params_norm:
+    value: true
+log_per_layer_grad_rms:
+    value: true
+log_per_module_grad_rms:
+    value: true
+log_per_module_update_rms:
+    value: false
+log_progress:
+    value: false
+log_spike_diagnostics:
+    value: false
+log_straggler:
+    value: false
+log_throughput:
+    value: true
+log_timers_to_tensorboard:
+    value: false
+log_update_rms:
+    value: true
+log_validation_ppl_to_tensorboard:
+    value: false
+log_world_size_to_tensorboard:
+    value: false
+logging_level:
+    value: null
+loss_scale:
+    value: null
+loss_scale_window:
+    value: 1000
+lr:
+    value: 0.0005
+lr_decay_iters:
+    value: null
+lr_decay_samples:
+    value: 877440
+lr_decay_style:
+    value: constant
+lr_warmup_fraction:
+    value: null
+lr_warmup_init:
+    value: 0
+lr_warmup_iters:
+    value: 0
+lr_warmup_samples:
+    value: 0
+lr_wsd_decay_iters:
+    value: null
+lr_wsd_decay_samples:
+    value: null
+lr_wsd_decay_style:
+    value: exponential
+main_grads_dtype:
+    value: torch.float32
+main_params_dtype:
+    value: torch.float32
+make_vocab_size_divisible_by:
+    value: 128
+mamba_disable_cp:
+    value: false
+mamba_expand:
+    value: 2
+mamba_head_dim:
+    value: 64
+mamba_num_groups:
+    value: 8
+mamba_num_heads:
+    value: null
+mamba_state_dim:
+    value: 128
+manual_gc:
+    value: false
+manual_gc_eval:
+    value: true
+manual_gc_interval:
+    value: 0
+mask_factor:
+    value: 1
+mask_prob:
+    value: 0.15
+mask_type:
+    value: random
+masked_softmax_fusion:
+    value: false
+max_position_embeddings:
+    value: 131072
+max_seqlen_per_cp_rank:
+    value: null
+max_tokens_to_oom:
+    value: 12000
+memory_snapshot_path:
+    value: null
+merge_file:
+    value: null
+micro_batch_size:
+    value: 1
+microbatch_group_size_per_vp_stage:
+    value: null
+mid_level_dataset_surplus:
+    value: 0.005
+min_loss_scale:
+    value: 1
+min_lr:
+    value: 0.0005
+min_offloaded_tensor_size:
+    value: 1048576
+mlp_chunks_for_prefill:
+    value: 1
+mmap_bin_files:
+    value: true
+mock_data:
+    value: false
+modelopt_enabled:
+    value: false
+moe_apply_probs_on_input:
+    value: false
+moe_aux_loss_coeff:
+    value: 0.05
+moe_deepep_num_sms:
+    value: 20
+moe_enable_deepep:
+    value: false
+moe_expert_capacity_factor:
+    value: null
+moe_extended_tp:
+    value: false
+moe_ffn_hidden_size:
+    value: 512
+moe_flex_dispatcher_backend:
+    value: deepep
+moe_gaux_overlap:
+    value: false
+moe_grouped_gemm:
+    value: true
+moe_hybridep_num_sms:
+    value: 16
+moe_input_jitter_eps:
+    value: null
+moe_layer_freq:
+    value:
+        - 0
+        - 0
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+        - 1
+moe_layer_recompute:
+    value: false
+moe_monitor_layers:
+    value: ""
+moe_monitor_log_interval:
+    value: null
+moe_pad_expert_input_to_capacity:
+    value: false
+moe_pad_experts_for_cuda_graph_inference:
+    value: false
+moe_per_layer_logging:
+    value: false
+moe_permute_backend:
+    value: auto
+moe_permute_fusion:
+    value: true
+moe_router_aux_loss_fusion:
+    value: auto
+moe_router_bias_update_method:
+    value: sign
+moe_router_bias_update_rate:
+    value: 0.001
+moe_router_dtype:
+    value: fp32
+moe_router_enable_expert_bias:
+    value: false
+moe_router_force_load_balancing:
+    value: false
+moe_router_fusion:
+    value: true
+moe_router_gemm_backend:
+    value: te
+moe_router_group_topk:
+    value: null
+moe_router_load_balancing_type:
+    value: global_aux_loss
+moe_router_num_groups:
+    value: null
+moe_router_padding_for_fp8:
+    value: false
+moe_router_padding_for_quantization:
+    value: false
+moe_router_pre_softmax:
+    value: false
+moe_router_score_function:
+    value: softmax
+moe_router_sqrt_gate:
+    value: true
+moe_router_topk:
+    value: 4
+moe_router_topk_scaling_factor:
+    value: null
+moe_shared_expert_gate:
+    value: true
+moe_shared_expert_intermediate_size:
+    value: 512
+moe_shared_expert_overlap:
+    value: true
+moe_token_dispatcher_type:
+    value: alltoall
+moe_token_drop_policy:
+    value: probs
+moe_triton_dispatcher_exclude_padding_tokens:
+    value: false
+moe_triton_mega_activation_sms:
+    value: 4
+moe_triton_mega_combine_sms:
+    value: 32
+moe_triton_mega_compute_sms:
+    value: 24
+moe_triton_mega_dispatch_sms:
+    value: 48
+moe_triton_mega_num_warps:
+    value: 16
+moe_triton_mega_reduce_sms:
+    value: 32
+moe_triton_mega_tail_sms:
+    value: 16
+moe_triton_shared_expert_overlap:
+    value: false
+moe_triton_workspace_expand:
+    value: 4
+moe_upcycling_granularity:
+    value: 1
+moe_use_legacy_grouped_gemm:
+    value: false
+moe_use_upcycling:
+    value: false
+moe_z_loss_coeff:
+    value: null
+mrope_section:
+    value: null
+mscale:
+    value: 1
+mscale_all_dim:
+    value: 0
+mtp_linear_attention_type:
+    value: gated_delta_net
+mtp_loss_scaling_factor:
+    value: 0.1
+mtp_num_layers:
+    value: 0
+multi_latent_attention:
+    value: false
+multiple_validation_sets:
+    value: false
+muon_ball_momentum:
+    value: 0.9
+muon_ball_msign_steps:
+    value: 5
+muon_ball_power_iteration_steps:
+    value: 10
+muon_ball_qkv_split_mode:
+    value: component
+muon_ball_radius_mode:
+    value: spectral_mup
+muon_ball_retract_alpha:
+    value: 0.05
+muon_ball_retract_mode:
+    value: hard
+muon_ball_scale_mode:
+    value: spectral_mup
+muon_ball_split_fc1:
+    value: true
+muon_ball_split_moe_experts:
+    value: true
+muon_ball_split_qkv:
+    value: true
+muon_ball_use_nesterov:
+    value: true
+muon_extra_scale_factor:
+    value: 1
+muon_fp32_matmul_prec:
+    value: medium
+muon_momentum:
+    value: 0.9
+muon_num_ns_steps:
+    value: 5
+muon_qkv_split_mode:
+    value: component
+muon_scale_mode:
+    value: spectral_mup
+muon_scale_vectorized_mode:
+    value: full
+muon_split_fc1:
+    value: true
+muon_split_moe_experts:
+    value: true
+muon_split_qkv:
+    value: true
+muon_tp_mode:
+    value: blockwise
+muon_use_nesterov:
+    value: false
+muon_vectorize:
+    value: []
+muon_vectorize_attn_dim:
+    value: hidden_size
+mup_lr_mult:
+    value: 1
+nccl_all_reduce_for_prefill:
+    value: false
+nccl_communicator_config_path:
+    value: null
+nccl_ub:
+    value: false
+no_load_optim:
+    value: null
+no_load_rng:
+    value: true
+no_load_scheduler:
+    value: null
+no_persist_layer_norm:
+    value: false
+no_rope_freq:
+    value: null
+no_save_optim:
+    value: null
+no_save_rng:
+    value: null
+no_save_step_one:
+    value: true
+no_weight_decay_cond_type:
+    value: null
+non_persistent_ckpt_type:
+    value: null
+non_persistent_global_ckpt_dir:
+    value: null
+non_persistent_keep_last:
+    value: 1
+non_persistent_local_ckpt_algo:
+    value: fully_parallel
+non_persistent_local_ckpt_dir:
+    value: null
+non_persistent_save_interval:
+    value: null
+norm_epsilon:
+    value: 1e-06
+normalization:
+    value: RMSNorm
+num_attention_heads:
+    value: 32
+num_channels:
+    value: 3
+num_classes:
+    value: 1000
+num_dataset_builder_threads:
+    value: 1
+num_distributed_optimizer_instances:
+    value: 1
+num_experts:
+    value: 192
+num_layers:
+    value: 56
+num_layers_at_end_in_bf16:
+    value: 1
+num_layers_at_start_in_bf16:
+    value: 1
+num_layers_per_virtual_pipeline_stage:
+    value: null
+num_query_groups:
+    value: 8
+num_virtual_stages_per_pipeline_rank:
+    value: null
+num_workers:
+    value: 4
+object_storage_cache_path:
+    value: null
+offload_modules:
+    value: []
+one_logger_async:
+    value: false
+one_logger_project:
+    value: megatron-lm
+one_logger_run_name:
+    value: null
+onnx_safe:
+    value: null
+openai_gelu:
+    value: false
+optimizer:
+    value: adam
+optimizer_cpu_offload:
+    value: false
+optimizer_offload_fraction:
+    value: 1
+output_bert_embeddings:
+    value: false
+output_logits_z_loss_coeff:
+    value: null
+overlap_cpu_optimizer_d2h_h2d:
+    value: false
+overlap_grad_reduce:
+    value: true
+overlap_moe_expert_parallel_comm:
+    value: false
+overlap_p2p_comm:
+    value: false
+overlap_p2p_comm_warmup_flush:
+    value: false
+overlap_param_gather:
+    value: true
+overlap_param_gather_with_optimizer_step:
+    value: false
+override_hf_eod_token_id:
+    value: null
+override_opt_param_scheduler:
+    value: true
+packing_tokens_per_shard:
+    value: 100000000000
+padded_vocab_size:
+    value: 99072
+params_dtype:
+    value: torch.bfloat16
+patch_dim:
+    value: 16
+per_split_data_args_path:
+    value: null
+perform_initialization:
+    value: false
+perform_rl_step:
+    value: false
+pin_cpu_grads:
+    value: true
+pin_cpu_params:
+    value: true
+pipeline_model_parallel_comm_backend:
+    value: null
+pipeline_model_parallel_layout:
+    value: null
+pipeline_model_parallel_size:
+    value: 1
+position_embedding_type:
+    value: none
+pp_aware_finegrained_recompute:
+    value: false
+pp_aware_finegrained_recompute_global_layers:
+    value: null
+pp_aware_finegrained_recompute_pp_ranks:
+    value: null
+prefetch_factor:
+    value: 8
+pretrained_checkpoint:
+    value: null
+previous_stage_seq_length:
+    value: null
+profile:
+    value: false
+profile_microbatch_count:
+    value: 2
+profile_microbatch_dir:
+    value: null
+profile_microbatch_iteration:
+    value: null
+profile_microbatch_start:
+    value: 0
+profile_ranks:
+    value:
+        - 0
+profile_step_end:
+    value: null
+profile_step_start:
+    value: null
+q_lora_rank:
+    value: null
+qk_head_dim:
+    value: 128
+qk_l2_norm:
+    value: false
+qk_layernorm:
+    value: false
+qk_pos_emb_head_dim:
+    value: 64
+query_in_block_prob:
+    value: 0.1
+quick_geglu:
+    value: false
+rampup_batch_size:
+    value: null
+rank:
+    value: 63
+recompute_granularity:
+    value: full
+recompute_method:
+    value: block
+recompute_modules:
+    value: null
+recompute_num_layers:
+    value: 56
+record_memory_history:
+    value: false
+relative_attention_max_distance:
+    value: 128
+relative_attention_num_buckets:
+    value: 32
+reparam_keys:
+    value: null
+replication:
+    value: false
+replication_factor:
+    value: 2
+replication_jump:
+    value: null
+rerun_mode:
+    value: validate_results
+reset_attention_mask:
+    value: false
+reset_iteration_one_to_zero:
+    value: false
+reset_position_ids:
+    value: false
+result_rejected_tracker_filename:
+    value: null
+retriever_report_topk_accuracies:
+    value: []
+retriever_score_scaling:
+    value: false
+retriever_seq_length:
+    value: 256
+retro_add_retriever:
+    value: false
+retro_attention_gate:
+    value: 1
+retro_cyclic_train_iters:
+    value: null
+retro_encoder_attention_dropout:
+    value: 0.1
+retro_encoder_hidden_dropout:
+    value: 0.1
+retro_encoder_layers:
+    value: 2
+retro_num_neighbors:
+    value: 2
+retro_num_retrieved_chunks:
+    value: 2
+retro_project_dir:
+    value: null
+retro_verify_neighbor_count:
+    value: true
+reuse_grad_buf_for_mxfp8_param_ag:
+    value: false
+reuse_training_communicators:
+    value: true
+rl_calculate_intra_group_similarity:
+    value: false
+rl_importance_sampling_truncation_coef:
+    value: null
+rl_inference_logprobs_is_correction:
+    value: false
+rl_offload_kv_cache_during_training:
+    value: false
+rl_offload_optimizer_during_inference:
+    value: false
+rl_partial_rollouts:
+    value: false
+rl_prompts_per_eval:
+    value: 32
+rl_remove_kv_cache_during_training:
+    value: false
+rl_reset_cuda_graphs:
+    value: false
+rl_sequence_packing_algo:
+    value: fifo
+rl_sequence_packing_bin_size:
+    value: 8192
+rl_use_sequence_packing:
+    value: false
+rope_scaling_factor:
+    value: 8
+rope_type:
+    value: null
+rotary_base:
+    value: 10000
+rotary_interleaved:
+    value: false
+rotary_percent:
+    value: 1
+rotary_scaling_factor:
+    value: 1
+rotary_seq_len_interpolation_factor:
+    value: null
+run_workload_inspector_server:
+    value: false
+runtime_audit_dir:
+    value: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit
+sample_rate:
+    value: 1
+save:
+    value: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints
+save_after_load:
+    value: false
+save_interval:
+    value: 40
+save_retain_interval:
+    value: 1000000000
+scale_emb:
+    value: 1
+scatter_gather_tensors_in_pipeline:
+    value: true
+seed:
+    value: 1234
+seq_length:
+    value: 131072
+sequence_packing_scheduler:
+    value: default_dynamic_cp
+sequence_parallel:
+    value: false
+sft:
+    value: false
+sft_tokenizer_prompt_format:
+    value: nemotron-h-aligned
+sgd_momentum:
+    value: 0.9
+sharp_enabled_group:
+    value: null
+short_seq_prob:
+    value: 0.1
+skip_train:
+    value: false
+skipped_train_samples:
+    value: 0
+slurm_job_name:
+    value: N/A
+softmax_type:
+    value: vanilla
+spec:
+    value: null
+spectra_clip:
+    value: 1
+spectra_grad_clip:
+    value: null
+spectra_hyperball:
+    value: false
+spectra_ns_steps:
+    value: 10
+spectra_scope:
+    value: matrices
+spectral_ball_momentum:
+    value: 0.9
+spectral_ball_msign_steps:
+    value: 8
+spectral_ball_power_iteration_steps:
+    value: 20
+spectral_ball_qkv_split_mode:
+    value: component
+spectral_ball_radius_mode:
+    value: spectral_mup
+spectral_ball_retract_alpha:
+    value: 0.05
+spectral_ball_retract_mode:
+    value: hard
+spectral_ball_scale_mode:
+    value: spectral_mup
+spectral_ball_solver:
+    value: bisection
+spectral_ball_solver_max_iterations:
+    value: 20
+spectral_ball_solver_tolerance_f:
+    value: 1e-08
+spectral_ball_split_fc1:
+    value: true
+spectral_ball_split_moe_experts:
+    value: true
+spectral_ball_split_qkv:
+    value: true
+spectral_ball_use_nesterov:
+    value: true
+spectral_mup_init:
+    value: false
+spike_diagnostics_dir:
+    value: null
+spike_diagnostics_end:
+    value: 21
+spike_diagnostics_start:
+    value: 2
+split:
+    value: 999,1,0
+split_expert_init:
+    value: true
+split_fc1_init:
+    value: true
+split_qkv_init:
+    value: true
+split_qkv_init_mode:
+    value: group
+sqreglu:
+    value: false
+squared_relu:
+    value: false
+start_samples:
+    value: 0
+start_weight_decay:
+    value: 0.1
+straggler_ctrlr_port:
+    value: 65535
+straggler_minmax_count:
+    value: 1
+strict_fsdp_dtensor_load:
+    value: true
+suggested_communication_unit_size:
+    value: null
+swanlab_exp_name:
+    value: ""
+swanlab_project:
+    value: ""
+swanlab_save_dir:
+    value: ""
+swanlab_workspace:
+    value: ""
+swiglu:
+    value: true
+swin_backbone_type:
+    value: tiny
+symmetric_ar_type:
+    value: null
+te_rng_tracker:
+    value: false
+teacher_model_config:
+    value: null
+tensor_model_parallel_size:
+    value: 1
+tensorboard_dir:
+    value: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard
+tensorboard_log_interval:
+    value: 1
+tensorboard_queue_size:
+    value: 1000
+test_data_path:
+    value: null
+test_mode:
+    value: false
+tiktoken_num_special_tokens:
+    value: 1000
+tiktoken_pattern:
+    value: null
+tiktoken_special_tokens:
+    value: null
+timing_log_level:
+    value: 0
+timing_log_option:
+    value: all
+titles_data_path:
+    value: null
+token_shift_conv_init:
+    value: default
+token_shift_conv_size:
+    value: 4
+tokenizer_metadata:
+    value: null
+tokenizer_model:
+    value: /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp
+tokenizer_type:
+    value: HuggingFaceTokenizer
+torch_fsdp2_reshard_after_forward:
+    value: true
+tp_comm_bootstrap_backend:
+    value: nccl
+tp_comm_bulk_dgrad:
+    value: true
+tp_comm_bulk_wgrad:
+    value: true
+tp_comm_overlap:
+    value: false
+tp_comm_overlap_ag:
+    value: true
+tp_comm_overlap_cfg:
+    value: null
+tp_comm_overlap_rs:
+    value: true
+tp_comm_overlap_rs_dgrad:
+    value: false
+tp_comm_split_ag:
+    value: true
+tp_comm_split_rs:
+    value: true
+train_data_path:
+    value: null
+train_iters:
+    value: null
+train_samples:
+    value: 877440
+train_sync_interval:
+    value: null
+train_until_stopped:
+    value: false
+transformer_impl:
+    value: transformer_engine
+transformer_pipeline_model_parallel_size:
+    value: 1
+trust_remote_code:
+    value: false
+untie_embeddings_and_output_weights:
+    value: true
+use_checkpoint_args:
+    value: false
+use_checkpoint_opt_param_scheduler:
+    value: false
+use_cpu_initialization:
+    value: null
+use_dist_ckpt:
+    value: true
+use_dist_ckpt_deprecated:
+    value: false
+use_distributed_optimizer:
+    value: true
+use_flash_attn:
+    value: true
+use_fused_weighted_squared_relu:
+    value: false
+use_legacy_models:
+    value: false
+use_megatron_fsdp:
+    value: false
+use_mp_args_from_checkpoint_args:
+    value: false
+use_one_sent_docs:
+    value: false
+use_persistent_ckpt_worker:
+    value: false
+use_precision_aware_optimizer:
+    value: false
+use_pytorch_profiler:
+    value: false
+use_ring_exchange_p2p:
+    value: false
+use_rope_scaling:
+    value: false
+use_rotary_position_embeddings:
+    value: false
+use_sharp:
+    value: false
+use_te_activation_func:
+    value: false
+use_tokenizer_model_from_checkpoint_args:
+    value: true
+use_torch_fsdp2:
+    value: false
+use_torch_optimizer_for_cpu_offload:
+    value: false
+use_tp_pp_dp_mapping:
+    value: false
+v_head_dim:
+    value: 128
+valid_data_path:
+    value: null
+variable_seq_lengths:
+    value: false
+virtual_pipeline_model_parallel_size:
+    value: null
+vision_backbone_type:
+    value: vit
+vision_pretraining:
+    value: false
+vision_pretraining_type:
+    value: classify
+vocab_extra_ids:
+    value: 0
+vocab_file:
+    value: null
+vocab_size:
+    value: null
+wandb_entity:
+    value: ""
+wandb_exp_name:
+    value: stage14_120B_8node_gdnlralpha_off_20260925a
+wandb_project:
+    value: qwen2.5_3b_mom_cpt
+wandb_save_dir:
+    value: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb
+weight_decay:
+    value: 0.1
+weight_decay_incr_style:
+    value: constant
+wgrad_deferral_limit:
+    value: 0
+window_attn_skip_freq:
+    value: null
+window_size:
+    value: null
+word_embedding_dropout_prob:
+    value: 0
+world_size:
+    value: 64
+yaml_cfg:
+    value: null

f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/requirements.txt

@@ -0,0 +1,202 @@
+jupyter_core==5.9.1
+triton==3.4.0
+decorator==5.2.1
+pybind11==3.0.1
+importlib_metadata==8.7.0
+joblib==1.5.2
+pillow==12.0.0
+multiprocess==0.70.18
+asttokens==3.0.1
+prettytable==3.17.0
+networkx==3.4.2
+aniso8601==10.0.1
+setuptools==69.0.0
+pandas==2.3.3
+kiwisolver==1.4.9
+aiohttp==3.13.2
+Flask-RESTful==0.3.10
+Jinja2==3.1.6
+urllib3==2.5.0
+hf-xet==1.5.1
+nltk==3.9.2
+fonttools==4.61.0
+triton_dist==3.4.0
+requests==2.32.5
+ipykernel==7.1.0
+multidict==6.7.0
+typer==0.25.1
+linghe==0.4.5
+pyparsing==3.2.5
+python-dateutil==2.9.0.post0
+pytest-random-order==1.2.0
+pydantic==2.12.5
+click==8.4.1
+safetensors==0.8.0
+openai==2.9.0
+Flask==3.1.2
+ipython==8.37.0
+pyzmq==27.1.0
+fla-core==0.5.0
+jmespath==1.1.0
+onnxscript==0.5.6
+nvidia-cudnn-cu12==9.10.2.21
+virtualenv-clone==0.5.7
+huggingface_hub==1.19.0
+nvidia-cuda-runtime-cu12==12.8.90
+nvitop==1.6.0
+transformer_engine==2.11.0.dev0+3ff0b8d4
+wrapt==2.0.1
+nvidia-ml-py==13.580.82
+coverage==7.12.0
+dill==0.4.0
+ml_dtypes==0.5.4
+s3transfer==0.19.2
+idna==3.18
+filelock==3.29.4
+pyarrow==22.0.0
+PyNaCl==1.6.2
+zarr==2.18.3
+gitdb==4.0.12
+sentencepiece==0.2.1
+annotated-doc==0.0.4
+tokenizers==0.22.2
+pexpect==4.9.0
+pytest-mock==3.15.1
+torchvision==0.23.0
+boto3==1.43.99
+tzdata==2025.2
+modelscope==1.32.0
+tomli==2.3.0
+jsonschema==4.26.0
+transformers==5.13.0
+markdown-it-py==4.2.0
+nvidia-cuda-nvrtc-cu12==12.8.93
+Werkzeug==3.1.4
+pip==25.3
+flash-linear-attention==0.5.0
+numpy==2.2.6
+wcwidth==0.2.14
+wheel==0.45.1
+iniconfig==2.3.0
+debugpy==1.8.17
+packaging==26.2
+zipp==3.23.0
+pluggy==1.6.0
+frozenlist==1.8.0
+jsonschema-specifications==2025.9.1
+exceptiongroup==1.3.1
+cffi==2.0.0
+nvidia-nvtx-cu12==12.8.90
+charset-normalizer==3.4.4
+accelerate==1.12.0
+nvidia-cusparse-cu12==12.5.8.93
+protobuf==6.33.1
+PyYAML==6.0.3
+mdurl==0.1.2
+torch==2.8.0
+six==1.17.0
+nvidia-nvshmem-cu12==3.3.9
+pytest==9.0.1
+mpmath==1.3.0
+xxhash==3.6.0
+nvidia-cufft-cu12==11.3.3.83
+typing_extensions==4.15.0
+attrs==25.4.0
+executing==2.2.1
+nvidia-cusparselt-cu12==0.7.1
+tiktoken==0.12.0
+nvidia-cuda-cupti-cu12==12.8.90
+sympy==1.14.0
+matplotlib==3.10.8
+rich==15.0.0
+nvidia-cusolver-cu12==11.7.3.90
+causal_conv1d==1.5.4
+simplejson==3.20.2
+prompt_toolkit==3.0.52
+typing-inspection==0.4.2
+MarkupSafe==3.0.3
+rpds-py==0.30.0
+nvidia-modelopt==0.39.0
+PyGithub==2.9.1
+contourpy==1.3.2
+jiter==0.12.0
+tqdm==4.68.2
+cycler==0.12.1
+nest-asyncio==1.6.0
+propcache==0.4.1
+pytest-cov==7.0.0
+cuda-core==0.2.0
+parso==0.8.5
+itsdangerous==2.2.0
+bracex==3.0.1
+nvidia-cufile-cu12==1.13.1.3
+jedi==0.19.2
+Pygments==2.20.0
+jupyter_client==8.6.3
+swanlab==0.7.5
+shellingham==1.5.4
+datasets==4.4.1
+nvidia-curand-cu12==10.3.9.90
+async-timeout==5.0.1
+smmap==5.0.2
+stack-data==0.6.3
+flash_attn==2.8.3
+tensorstore==0.1.78
+referencing==0.37.0
+matplotlib-inline==0.2.1
+einops==0.8.2
+certifi==2026.5.20
+mamba_ssm==2.2.6.post3
+pure_eval==0.2.3
+aiosignal==1.4.0
+sniffio==1.3.1
+Cython==0.29.24
+pydantic_core==2.41.5
+onnx==1.19.1
+onnx-ir==0.1.12
+annotated-types==0.7.0
+xattr==1.3.0
+wcmatch==10.2.1
+torchprofile==0.0.4
+psutil==7.1.3
+platformdirs==4.5.0
+sentry-sdk==2.46.0
+PyJWT==2.13.0
+ptyprocess==0.7.0
+nvidia-cublas-cu12==12.8.4.1
+PuLP==3.3.0
+apex==0.1
+scipy==1.15.3
+comm==0.2.3
+tornado==6.5.2
+traitlets==5.14.3
+distro==1.9.0
+yarl==1.22.0
+GitPython==3.1.45
+ninja==1.13.0
+blinker==1.9.0
+httpcore==1.0.9
+nvidia-nccl-cu12==2.27.3
+multi-storage-client==0.51.0
+nvidia-nvjitlink-cu12==12.8.93
+httpx==0.28.1
+fasteners==0.20
+h11==0.16.0
+wandb==0.23.0
+torchaudio==2.8.0
+aiohappyeyeballs==2.6.1
+numcodecs==0.13.1
+nvshmem4py-cu12==0.1.2
+opentelemetry-api==1.44.0
+asciitree==0.3.3
+anyio==4.14.0
+botocore==1.43.99
+fsspec==2026.6.0
+absl-py==2.3.1
+pytz==2025.2
+cryptography==49.0.0
+lark==1.3.1
+regex==2026.5.9
+pycparser==3.0
+pyecharts==2.0.9
+cuda-python==12.4.0

f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/wandb-metadata.json

@@ -0,0 +1,384 @@
+{
+  "os": "Linux-5.15.0-94-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.10.19",
+  "startedAt": "2026-10-03T07:36:55.078128Z",
+  "args": [
+    "--use-mcore-models",
+    "--hybrid-attention-ratio",
+    "0.0625",
+    "--hybrid-mlp-ratio",
+    "0.5",
+    "--is-hybrid-model",
+    "--linear-attention-type",
+    "gated_delta_net",
+    "--linear-conv-kernel-dim",
+    "4",
+    "--linear-key-head-dim",
+    "64",
+    "--linear-value-head-dim",
+    "64",
+    "--linear-num-key-heads",
+    "8",
+    "--linear-num-value-heads",
+    "32",
+    "--disable-bias-linear",
+    "--seq-length",
+    "131072",
+    "--max-position-embeddings",
+    "131072",
+    "--num-layers",
+    "56",
+    "--hidden-size",
+    "2048",
+    "--ffn-hidden-size",
+    "4800",
+    "--num-attention-heads",
+    "32",
+    "--init-method-std",
+    "0.02",
+    "--attention-dropout",
+    "0.0",
+    "--hidden-dropout",
+    "0.0",
+    "--normalization",
+    "RMSNorm",
+    "--swiglu",
+    "--group-query-attention",
+    "--num-query-groups",
+    "8",
+    "--no-masked-softmax-fusion",
+    "--use-flash-attn",
+    "--add-qkv-bias",
+    "--mtp-num-layers",
+    "0",
+    "--mtp-loss-scaling-factor",
+    "0.1",
+    "--norm-epsilon",
+    "1e-06",
+    "--mtp-linear-attention-type",
+    "gated_delta_net",
+    "--position-embedding-type",
+    "none",
+    "--linear-attention-freq",
+    "[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]",
+    "--optimizer",
+    "adam",
+    "--adam-beta1",
+    "0.9",
+    "--adam-beta2",
+    "0.95",
+    "--adam-eps",
+    "1e-15",
+    "--weight-decay",
+    "0.1",
+    "--clip-grad",
+    "1.0",
+    "--data-path",
+    "/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp",
+    "--split",
+    "999,1,0",
+    "--data-cache-path",
+    "/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2",
+    "--no-create-attention-mask-in-dataloader",
+    "--num-experts",
+    "192",
+    "--moe-grouped-gemm",
+    "--moe-router-topk",
+    "4",
+    "--moe-router-score-function",
+    "softmax",
+    "--overlap-param-gather",
+    "--overlap-grad-reduce",
+    "--moe-expert-capacity-factor",
+    "0",
+    "--moe-shared-expert-intermediate-size",
+    "512",
+    "--moe-router-dtype",
+    "fp32",
+    "--moe-permute-fusion",
+    "--moe-ffn-hidden-size",
+    "512",
+    "--moe-shared-expert-gate",
+    "--moe-shared-expert-overlap",
+    "--moe-layer-freq",
+    "([0]*2+[1]*54)",
+    "--moe-aux-loss-coeff",
+    "0.05",
+    "--moe-router-load-balancing-type",
+    "global_aux_loss",
+    "--micro-batch-size",
+    "1",
+    "--global-batch-size",
+    "128",
+    "--lr",
+    "5e-4",
+    "--train-samples",
+    "877440",
+    "--lr-warmup-samples",
+    "0",
+    "--lr-decay-samples",
+    "877440",
+    "--lr-decay-style",
+    "constant",
+    "--min-lr",
+    "0.0005",
+    "--bf16",
+    "--async-save",
+    "--ckpt-fully-parallel-load",
+    "--auto-detect-ckpt-format",
+    "--load-complemental-dataset",
+    "loss_mask",
+    "--tensor-model-parallel-size",
+    "1",
+    "--pipeline-model-parallel-size",
+    "1",
+    "--context-parallel-size",
+    "8",
+    "--expert-model-parallel-size",
+    "8",
+    "--sequence-parallel",
+    "--use-distributed-optimizer",
+    "--log-interval",
+    "1",
+    "--log-throughput",
+    "--save-interval",
+    "40",
+    "--eval-interval",
+    "99999999",
+    "--eval-iters",
+    "10",
+    "--increase-log-level-iters",
+    "5",
+    "--timing-log-level",
+    "0",
+    "--wandb-project",
+    "qwen2.5_3b_mom_cpt",
+    "--tokenizer-type",
+    "HuggingFaceTokenizer",
+    "--tokenizer-model",
+    "/mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp",
+    "--no-adam-bias-correction",
+    "--corrected-weight-decay",
+    "--auto-generate-cu-seqlens",
+    "--cp-comm-type",
+    "a2a",
+    "--log-moe-activations",
+    "--final-logits-clamp-value",
+    "30",
+    "--attention-output-gate",
+    "--gated-norm",
+    "--gated-norm-lora-rank",
+    "16",
+    "--dataloader-type",
+    "cyclic",
+    "--lm-loss-normalization",
+    "global-token",
+    "--prefetch-factor",
+    "8",
+    "--activation-func-clamp-value",
+    "7",
+    "--activation-func-clamp-mode",
+    "soft",
+    "--gdn-split-in-proj",
+    "--gdn-fused-split-forward",
+    "--gdn-alpha-weight-scale",
+    "1.0",
+    "--gdn-alpha-weight-scale-mode",
+    "matrix",
+    "--gdn-lr-mult-alpha",
+    "0.5",
+    "--start-samples",
+    "0",
+    "--ckpt-format",
+    "torch_dist",
+    "--rerun-mode",
+    "validate_results",
+    "--distributed-timeout-minutes",
+    "120",
+    "--dist-ckpt-strictness",
+    "assume_ok_unexpected",
+    "--gdn-backend",
+    "fla",
+    "--gdn-gated-norm-backend",
+    "linghe",
+    "--moe-router-gemm-backend",
+    "te",
+    "--log-params-norm",
+    "--timing-log-option",
+    "all",
+    "--no-barrier-with-level-1-timing",
+    "--log-metrics-interval",
+    "10",
+    "--cross-entropy-loss-fusion",
+    "--cross-entropy-fusion-impl",
+    "linear_triton",
+    "--linear-cross-entropy-num-chunks",
+    "8",
+    "--num-workers",
+    "4",
+    "--moe-router-sqrt-gate",
+    "--moe-router-fusion",
+    "--moe-token-dispatcher-type",
+    "alltoall",
+    "--log-hidden-states",
+    "embeddings",
+    "residual_attn",
+    "input_layernorm",
+    "attention::q_layernorm",
+    "attention::k_layernorm",
+    "attention::core_attention",
+    "attention::o_proj",
+    "residual_mlp",
+    "pre_mlp_layernorm",
+    "mlp",
+    "moe::router",
+    "moe::shared_experts",
+    "moe::routed_experts",
+    "gated_delta::in_proj",
+    "gated_delta::conv1d",
+    "gated_delta::query",
+    "gated_delta::key",
+    "gated_delta::value",
+    "gated_delta::gate",
+    "gated_delta::beta",
+    "gated_delta::alpha",
+    "gated_delta::g",
+    "gated_delta::core_attention",
+    "gated_delta::out_norm",
+    "gated_delta::out_proj",
+    "final_layernorm",
+    "--log-params",
+    "attention::linear_qkv",
+    "attention::o_proj",
+    "mlp::linear_fc1",
+    "mlp::linear_fc2",
+    "input_layernorm",
+    "pre_mlp_layernorm",
+    "embedding",
+    "lm_head",
+    "gated_delta::in_proj",
+    "gated_delta::in_proj_bias",
+    "gated_delta::conv1d",
+    "gated_delta::conv1d_bias",
+    "gated_delta::dt_bias",
+    "gated_delta::A_log",
+    "gated_delta::out_norm",
+    "gated_delta::out_norm_bias",
+    "gated_delta::out_proj",
+    "gated_delta::out_proj_bias",
+    "--log-update-rms",
+    "--log-per-module-grad-rms",
+    "--log-per-layer-grad-rms",
+    "--document-packing-algorithm",
+    "obfd",
+    "--exit-on-missing-checkpoint",
+    "--no-initialization",
+    "--untie-embeddings-and-output-weights",
+    "--recompute-granularity",
+    "full",
+    "--recompute-method",
+    "block",
+    "--recompute-num-layers",
+    "56",
+    "--no-save-step-one",
+    "--save-retain-interval",
+    "1000000000",
+    "--exit-signal-handler",
+    "--fast-thd-a2a",
+    "--hybrid-adamh",
+    "--reuse-training-communicators",
+    "--ddp-bucket-size",
+    "40000000",
+    "--load",
+    "/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints",
+    "--continual-pretrain",
+    "--save",
+    "/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints",
+    "--tensorboard-dir",
+    "/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard",
+    "--wandb-save-dir",
+    "/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb",
+    "--wandb-exp-name",
+    "stage14_120B_8node_gdnlralpha_off_20260925a",
+    "--runtime-audit-dir",
+    "/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit"
+  ],
+  "program": "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py",
+  "email": "huyiwen0301@hotmail.com",
+  "root": "/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb",
+  "host": "cs-3554a-53569-server",
+  "executable": "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python",
+  "cpu_count": 64,
+  "cpu_count_logical": 128,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "31526391808",
+      "used": "1319878656"
+    }
+  },
+  "memory": {
+    "total": "1081804955648"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-40999d1b-2b80-14f4-48d7-334e7dc3e650"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-c854c8e4-7352-b317-f9e8-7ab1d7a12c08"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-f7d2842f-48e9-649c-55a4-4efe7a6e20f7"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-03925119-14d4-a11c-d740-0ba854285052"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-7cbabdfb-9388-3fbd-babc-d0b03ac9f1bb"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-4999b9f4-9851-ce6a-4296-c230e2813b5f"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-9e23b382-2f8e-e424-f025-c2b311d96c54"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-4241d9ca-7546-c1c3-dba8-bdc5786a3777"
+    }
+  ],
+  "cudaVersion": "13.3",
+  "writerId": "i4bo5dz9n6n9rgehz11zfcuyno7peo6e"
+}

f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/logs/debug-internal.log

@@ -0,0 +1,116 @@
+{"time":"2026-10-03T15:36:55.391747798+08:00","level":"INFO","msg":"stream: starting","core version":"0.23.0"}
+{"time":"2026-10-03T15:36:56.343971627+08:00","level":"INFO","msg":"stream: created new stream","id":"neb76hvj"}
+{"time":"2026-10-03T15:36:56.344046883+08:00","level":"INFO","msg":"handler: started","stream_id":"neb76hvj"}
+{"time":"2026-10-03T15:36:56.344730971+08:00","level":"INFO","msg":"stream: started","id":"neb76hvj"}
+{"time":"2026-10-03T15:36:56.344743257+08:00","level":"INFO","msg":"writer: started","stream_id":"neb76hvj"}
+{"time":"2026-10-03T15:36:56.344739432+08:00","level":"INFO","msg":"sender: started","stream_id":"neb76hvj"}
+{"time":"2026-10-03T20:46:23.552125568+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T20:46:36.077615671+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T20:48:50.064249591+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": dial tcp 34.8.250.101:443: connect: connection timed out"}
+{"time":"2026-10-03T20:49:08.79949489+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T20:54:11.796774742+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T20:54:25.486179586+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T20:54:40.545937717+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T20:55:00.580305387+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T20:55:28.634028057+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T20:56:17.718812384+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T20:56:30.19182662+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T20:56:45.115099873+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T20:57:04.950041949+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T20:57:33.290989029+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T20:58:17.249014078+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:18:07.6748799+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:18:20.477958703+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:26:22.679695426+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:26:52.680964647+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:27:05.58031186+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:27:20.891877642+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:27:39.777644747+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:28:07.705963999+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:28:55.980689331+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:46:22.751047592+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:46:35.471208755+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:46:50.725116709+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:48:13.853271803+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:48:26.620263443+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:48:41.252419412+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T21:48:59.941891806+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T22:23:17.148564204+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T22:24:34.961977946+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T22:24:49.868897624+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T22:25:09.837019477+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T22:25:22.740851635+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T22:25:41.180815099+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T22:26:02.099461715+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T23:16:37.70080957+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T23:16:50.079410395+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T23:17:06.416669273+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T23:17:19.195204077+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T23:17:33.957777145+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T23:17:56.940595699+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T23:19:15.017046246+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-03T23:19:28.467688489+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T00:16:56.18210566+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T00:17:09.315476895+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T00:17:21.940223945+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T00:19:48.767396206+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T00:20:02.042044451+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T00:20:17.40407131+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T00:20:29.883880501+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T00:20:30.485669138+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-04T00:20:44.994579313+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T00:21:02.900640914+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-04T00:21:37.575387697+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-04T00:22:16.426157864+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-04T00:22:57.246240087+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":6002}
+{"time":"2026-10-04T00:23:04.708055607+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-04T00:23:40.822674886+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":10}
+{"time":"2026-10-04T00:51:37.946835389+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T00:53:50.608248186+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": dial tcp 34.8.250.101:443: connect: connection timed out"}
+{"time":"2026-10-04T00:54:21.932925999+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:21:08.268580691+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:21:20.616873937+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:21:34.684804207+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": EOF"}
+{"time":"2026-10-04T20:21:47.257725082+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": EOF"}
+{"time":"2026-10-04T20:22:22.001280214+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-04T20:22:40.639904893+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:23:00.036970039+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:23:29.015889448+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:24:12.722170873+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:25:23.128186804+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:26:35.250621719+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:26:47.734909018+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:27:04.853048393+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:27:17.427268502+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:28:38.822647668+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:28:53.402251459+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-04T20:29:06.242458788+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T09:35:42.961267638+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream","body":"{\"error\":\"mysql query exceeded guard deadline: context deadline exceeded\"}"}
+{"time":"2026-10-05T09:35:43.003831834+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-05T09:36:15.387744012+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-05T09:36:34.472084593+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"error\":\"An internal error occurred. Please contact support.\"}"}
+{"time":"2026-10-05T09:36:41.311002027+08:00","level":"INFO","msg":"api: retrying HTTP error","status":429,"url":"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream","body":"{\"error\":\"rate limit exceeded: per_run limit on filestream requests\"}"}
+{"time":"2026-10-05T14:23:22.622411251+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T14:23:35.332510685+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T14:23:50.703125116+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T14:24:22.554969727+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T14:24:37.512289017+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T14:24:50.156339635+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T14:25:05.235963369+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T14:25:24.075085054+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T14:27:07.531449591+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T14:27:22.510235052+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T14:27:34.977708119+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T14:27:49.525380822+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T14:27:55.277339507+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": EOF"}
+{"time":"2026-10-05T14:28:08.330067714+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"}
+{"time":"2026-10-05T14:30:36.176808349+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": dial tcp 34.8.250.101:443: connect: connection timed out"}
+{"time":"2026-10-06T22:09:53.669948878+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T22:10:20.275442496+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-artifacts-prod/wandb_artifacts/1494869847/3827992217/f231f2107df69eab0a3862d50018a9b2?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer%40wandb-production.iam.gserviceaccount.com%2F20261006%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20261006T141019Z&X-Goog-Expires=86399&X-Goog-Signature=7d6629d602b0fcb4db2bc36e31fe1906e480dc486be14eae1b6514e00b2c934e1b88c80a7127704af63f2a3a549c813d2c64769e0515685c7fdbce6b005ed2eb4537bdf8ce3f07202d44d240e95df251274072e0103c4bfc9e70b14249d1ea36bfb3d7c64d42545990bd37c9a3139ce9926289be2dfde50f936a2030740769beb7054ec0291b9d5e900b36d785c2b71eb458fb728007628e9dce2a7b9a57216fa1506aced48542e228db7f6652a506ec275ea39f956dcfa4ad594e04601b1a87bbb442704eb767c3e484f7ec602a4b0990ad5fc6d47bf151ef35da4d8caabfc4f3b3503f81f2c172b86f28efcc5b1e2266a5a273d9f1c7bdaa4e6cc09cc21370&X-Goog-SignedHeaders=content-md5%3Bcontent-type%3Bhost&X-User=huyiwen\": read tcp 10.233.98.253:54508->142.250.73.155:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-artifacts-prod/wandb_artifacts/1494869847/3827992217/f231f2107df69eab0a3862d50018a9b2?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer%40wandb-production.iam.gserviceaccount.com%2F20261006%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20261006T141019Z&X-Goog-Expires=86399&X-Goog-Signature=7d6629d602b0fcb4db2bc36e31fe1906e480dc486be14eae1b6514e00b2c934e1b88c80a7127704af63f2a3a549c813d2c64769e0515685c7fdbce6b005ed2eb4537bdf8ce3f07202d44d240e95df251274072e0103c4bfc9e70b14249d1ea36bfb3d7c64d42545990bd37c9a3139ce9926289be2dfde50f936a2030740769beb7054ec0291b9d5e900b36d785c2b71eb458fb728007628e9dce2a7b9a57216fa1506aced48542e228db7f6652a506ec275ea39f956dcfa4ad594e04601b1a87bbb442704eb767c3e484f7ec602a4b0990ad5fc6d47bf151ef35da4d8caabfc4f3b3503f81f2c172b86f28efcc5b1e2266a5a273d9f1c7bdaa4e6cc09cc21370&X-Goog-SignedHeaders=content-md5%3Bcontent-type%3Bhost&X-User=huyiwen"}
+{"time":"2026-10-10T02:17:49.028881163+08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2026-10-10T20:25:50.740816179+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-10-10T20:25:53.530647034+08:00","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-10-10T20:25:53.586303464+08:00","level":"INFO","msg":"stream: closing","id":"neb76hvj"}
+{"time":"2026-10-10T20:25:53.586544397+08:00","level":"INFO","msg":"handler: closed","stream_id":"neb76hvj"}
+{"time":"2026-10-10T20:25:53.587170191+08:00","level":"INFO","msg":"sender: closed","stream_id":"neb76hvj"}
+{"time":"2026-10-10T20:25:53.587575033+08:00","level":"INFO","msg":"stream: closed","id":"neb76hvj"}

f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/logs/debug.log

@@ -0,0 +1,27 @@
+2026-10-03 15:36:55,089 INFO    MainThread:730851 [wandb_setup.py:_flush():80] Current SDK version is 0.23.0
+2026-10-03 15:36:55,090 INFO    MainThread:730851 [wandb_setup.py:_flush():80] Configure stats pid to 730851
+2026-10-03 15:36:55,090 INFO    MainThread:730851 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
+2026-10-03 15:36:55,090 INFO    MainThread:730851 [wandb_setup.py:_flush():80] Loading settings from /root/wandb/settings
+2026-10-03 15:36:55,090 INFO    MainThread:730851 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2026-10-03 15:36:55,090 INFO    MainThread:730851 [wandb_init.py:setup_run_log_directory():713] Logging user logs to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/logs/debug.log
+2026-10-03 15:36:55,090 INFO    MainThread:730851 [wandb_init.py:setup_run_log_directory():714] Logging internal logs to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/logs/debug-internal.log
+2026-10-03 15:36:55,090 INFO    MainThread:730851 [wandb_init.py:init():840] calling init triggers
+2026-10-03 15:36:55,091 INFO    MainThread:730851 [wandb_init.py:init():845] wandb.init called with sweep_config: {}
+config: {'num_layers': 56, 'encoder_num_layers': 56, 'decoder_num_layers': None, 'hidden_size': 2048, 'ffn_hidden_size': 4800, 'num_attention_heads': 32, 'attention_backend': <AttnBackend.auto: 5>, 'kv_channels': 64, 'group_query_attention': True, 'num_query_groups': 8, 'attention_output_gate': True, 'attention_output_gate_upcycling': False, 'gated_norm': True, 'gated_norm_upcycling': False, 'gated_norm_lora_rank': 16, 'softmax_type': 'vanilla', 'window_size': None, 'window_attn_skip_freq': None, 'max_position_embeddings': 131072, 'position_embedding_type': 'none', 'relative_attention_num_buckets': 32, 'relative_attention_max_distance': 128, 'use_rotary_position_embeddings': False, 'rotary_base': 10000, 'rotary_percent': 1.0, 'rotary_interleaved': False, 'rotary_seq_len_interpolation_factor': None, 'use_rope_scaling': False, 'rope_scaling_factor': 8.0, 'no_rope_freq': None, 'add_position_embedding': True, 'mrope_section': None, 'make_vocab_size_divisible_by': 128, 'normalization': 'RMSNorm', 'norm_epsilon': 1e-06, 'apply_layernorm_1p': False, 'freeze_layernorm_weight': False, 'scale_emb': 1.0, 'hidden_states_shrink': 1.0, 'lm_head_alpha': 1.0, 'mup_lr_mult': 1.0, 'apply_residual_connection_post_layernorm': False, 'openai_gelu': False, 'squared_relu': False, 'swiglu': True, 'quick_geglu': False, 'activation_func_clamp_value': 7.0, 'activation_func_clamp_mode': 'soft', 'glu_linear_offset': 0.0, 'sqreglu': False, 'geglu': False, 'onnx_safe': None, 'bert_binary_head': True, 'untie_embeddings_and_output_weights': True, 'multi_latent_attention': False, 'mtp_num_layers': 0, 'mtp_loss_scaling_factor': 0.1, 'mtp_linear_attention_type': 'gated_delta_net', 'ffn_token_shift': None, 'ffn_intermediate_token_shift': None, 'token_shift_conv_size': 4, 'token_shift_conv_init': 'default', 'attn_token_shift': None, 'attn_q_token_shift': None, 'attn_k_token_shift': None, 'attn_v_token_shift': None, 'attn_output_gate': None, 'emb_deviation_loss_coeff': 0, 'emb_deviation_type': None, 'reparam_keys': None, 'attention_dropout': 0.0, 'hidden_dropout': 0.0, 'word_embedding_dropout_prob': 0.0, 'weight_decay': 0.1, 'start_weight_decay': 0.1, 'end_weight_decay': 0.1, 'weight_decay_incr_style': 'constant', 'corrected_weight_decay': True, 'no_weight_decay_cond_type': None, 'clip_grad': 1.0, 'log_per_module_update_rms': False, 'log_spike_diagnostics': False, 'spike_diagnostics_start': 2, 'spike_diagnostics_end': 21, 'spike_diagnostics_dir': None, 'log_per_module_grad_rms': True, 'log_per_layer_grad_rms': True, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-15, 'adamw_lr_mup_scaler': False, 'sgd_momentum': 0.9, 'adamh_scope': 'lm_head', 'adamh_radius': None, 'adamh_eps': 1e-08, 'spectra_scope': 'matrices', 'spectra_hyperball': False, 'spectra_clip': 1.0, 'spectra_grad_clip': None, 'spectra_ns_steps': 10, 'muon_momentum': 0.9, 'muon_split_qkv': True, 'muon_qkv_split_mode': 'component', 'muon_split_fc1': True, 'muon_use_nesterov': False, 'muon_scale_mode': 'spectral_mup', 'muon_fp32_matmul_prec': 'medium', 'muon_num_ns_steps': 5, 'muon_tp_mode': 'blockwise', 'muon_extra_scale_factor': 1.0, 'muon_split_moe_experts': True, 'muon_vectorize': [], 'muon_vectorize_attn_dim': 'hidden_size', 'muon_scale_vectorized_mode': 'full', 'muon_ball_momentum': 0.9, 'muon_ball_use_nesterov': True, 'muon_ball_split_qkv': True, 'muon_ball_qkv_split_mode': 'component', 'muon_ball_split_fc1': True, 'muon_ball_split_moe_experts': True, 'muon_ball_msign_steps': 5, 'muon_ball_radius_mode': 'spectral_mup', 'muon_ball_power_iteration_steps': 10, 'muon_ball_scale_mode': 'spectral_mup', 'muon_ball_retract_mode': 'hard', 'muon_ball_retract_alpha': 0.05, 'spectral_ball_momentum': 0.9, 'spectral_ball_use_nesterov': True, 'spectral_ball_split_qkv': True, 'spectral_ball_qkv_split_mode': 'component', 'spectral_ball_split_fc1': True, 'spectral_ball_split_moe_experts': True, 'spectral_ball_msign_steps': 8, 'spectral_ball_solver': 'bisection', 'spectral_ball_solver_tolerance_f': 1e-08, 'spectral_ball_solver_max_iterations': 20, 'spectral_ball_radius_mode': 'spectral_mup', 'spectral_ball_power_iteration_steps': 20, 'spectral_ball_scale_mode': 'spectral_mup', 'spectral_ball_retract_mode': 'hard', 'spectral_ball_retract_alpha': 0.05, 'fast_thd_a2a': True, 'hybrid_adamh': True, 'reuse_training_communicators': True, 'runtime_audit_dir': '/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit', 'train_until_stopped': False, 'micro_batch_size': 1, 'global_batch_size': 128, 'rampup_batch_size': None, 'decrease_batch_size_if_needed': False, 'recompute_granularity': 'full', 'check_for_nan_in_loss_and_grad': True, 'check_for_spiky_loss': False, 'check_for_large_grads': False, 'distribute_saved_activations': False, 'recompute_method': 'block', 'recompute_num_layers': 56, 'pp_aware_finegrained_recompute': False, 'pp_aware_finegrained_recompute_pp_ranks': None, 'pp_aware_finegrained_recompute_global_layers': None, 'recompute_modules': None, 'clone_scatter_output_in_embedding': True, 'profile': False, 'profile_step_start': None, 'profile_step_end': None, 'iterations_to_skip': [], 'result_rejected_tracker_filename': None, 'enable_gloo_process_groups': True, 'use_pytorch_profiler': False, 'profile_ranks': [0], 'profile_microbatch_iteration': None, 'profile_microbatch_start': 0, 'profile_microbatch_count': 2, 'profile_microbatch_dir': None, 'record_memory_history': False, 'memory_snapshot_path': None, 'tp_comm_overlap': False, 'tp_comm_overlap_cfg': None, 'tp_comm_overlap_ag': True, 'tp_comm_overlap_rs': True, 'tp_comm_overlap_rs_dgrad': False, 'tp_comm_bulk_dgrad': True, 'tp_comm_bulk_wgrad': True, 'tp_comm_bootstrap_backend': 'nccl', 'use_cpu_initialization': None, 'empty_unused_memory_level': 0, 'deterministic_mode': False, 'check_weight_hash_across_dp_replicas_interval': None, 'calculate_per_token_loss': False, 'lm_loss_normalization': 'global-token', 'train_sync_interval': None, 'train_iters': None, 'train_samples': 877440, 'log_interval': 1, 'exit_interval': None, 'exit_duration_in_mins': None, 'exit_signal_handler': True, 'tensorboard_dir': '/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard', 'masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'bias_swiglu_fusion': True, 'use_fused_weighted_squared_relu': False, 'bias_dropout_fusion': True, 'apply_rope_fusion': False, 'rope_type': None, 'cross_entropy_loss_fusion': True, 'cross_entropy_fusion_impl': 'linear_triton', 'linear_cross_entropy_num_chunks': 8, 'final_logits_clamp_value': 30.0, 'linear_cross_entropy_accumulate_grad_in_fp32': True, 'use_flash_attn': True, 'add_bias_linear': False, 'add_qkv_bias': True, 'optimizer': 'adam', 'adam_bias_correction': False, 'optimizer_cpu_offload': False, 'optimizer_offload_fraction': 1.0, 'use_torch_optimizer_for_cpu_offload': False, 'overlap_cpu_optimizer_d2h_h2d': False, 'dump_param_to_param_group_map': None, 'pin_cpu_grads': True, 'pin_cpu_params': True, 'dataloader_type': 'cyclic', 'async_tensor_model_parallel_allreduce': True, 'no_persist_layer_norm': False, 'sequence_parallel': False, 'gradient_accumulation_fusion': True, 'deprecated_use_mcore_models': True, 'use_legacy_models': False, 'manual_gc': False, 'manual_gc_interval': 0, 'manual_gc_eval': True, 'tp_comm_split_ag': True, 'tp_comm_split_rs': True, 'pipeline_model_parallel_comm_backend': None, 'high_priority_stream_groups': [], 'use_te_activation_func': False, 'freeze_non_mamba': False, 'reset_iteration_one_to_zero': False, 'fine_grained_activation_offloading': False, 'offload_modules': [], 'min_offloaded_tensor_size': 1048576, 'perform_rl_step': False, 'rl_prompts_per_eval': 32, 'grpo_prompts_per_step': 32, 'grpo_group_size': 2, 'grpo_iterations': 2, 'grpo_clamp_eps_lower': 0.01, 'grpo_clamp_eps_upper': 0.01, 'grpo_kl_beta': 0.001, 'grpo_entropy_term_weight': 0.0, 'grpo_filter_groups_with_same_reward': False, 'grpo_default_temperature': 1.0, 'grpo_default_top_p': 0, 'langrl_inference_server_type': 'inplace_megatron', 'langrl_inference_server_conversation_template': None, 'langrl_external_server': False, 'langrl_env_config': None, 'rl_offload_optimizer_during_inference': False, 'rl_offload_kv_cache_during_training': False, 'rl_remove_kv_cache_during_training': False, 'rl_reset_cuda_graphs': False, 'rl_partial_rollouts': False, 'rl_inference_logprobs_is_correction': False, 'rl_importance_sampling_truncation_coef': None, 'rl_calculate_intra_group_similarity': False, 'rl_use_sequence_packing': False, 'rl_sequence_packing_bin_size': 8192, 'rl_sequence_packing_algo': 'fifo', 'seed': 1234, 'data_parallel_random_init': False, 'init_method_std': 0.02, 'split_qkv_init': True, 'split_qkv_init_mode': 'group', 'split_fc1_init': True, 'split_expert_init': True, 'embedding_init_method_std': None, 'spectral_mup_init': False, 'init_method_xavier_uniform': False, 'lr': 0.0005, 'lr_decay_style': 'constant', 'lr_wsd_decay_style': 'exponential', 'lr_decay_iters': None, 'lr_decay_samples': 877440, 'lr_wsd_decay_samples': None, 'lr_wsd_decay_iters': None, 'lr_warmup_fraction': None, 'lr_warmup_iters': 0, 'lr_warmup_samples': 0, 'lr_warmup_init': 0.0, 'min_lr': 0.0005, 'override_opt_param_scheduler': True, 'use_checkpoint_opt_param_scheduler': False, 'decoupled_lr': None, 'decoupled_min_lr': None, 'save': '/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints', 'no_save_step_one': True, 'save_interval': 40, 'save_retain_interval': 1000000000, 'no_save_optim': None, 'no_save_rng': None, 'load': '/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints', 'no_load_optim': None, 'load_main_params_from_ckpt': None, 'no_load_rng': True, 'no_load_scheduler': None, 'save_after_load': False, 'strict_fsdp_dtensor_load': True, 'non_persistent_save_interval': None, 'non_persistent_keep_last': 1, 'non_persistent_ckpt_type': None, 'non_persistent_global_ckpt_dir': None, 'non_persistent_local_ckpt_dir': None, 'non_persistent_local_ckpt_algo': 'fully_parallel', 'finetune': False, 'continual_pretrain': True, 'continual_pretrain_preserve_opt_param_scheduler': False, 'pretrained_checkpoint': None, 'ckpt_step': None, 'perform_initialization': False, 'use_checkpoint_args': False, 'use_mp_args_from_checkpoint_args': False, 'use_tokenizer_model_from_checkpoint_args': True, 'exit_on_missing_checkpoint': True, 'use_dist_ckpt_deprecated': False, 'use_persistent_ckpt_worker': False, 'auto_detect_ckpt_format': True, 'dist_ckpt_format_deprecated': None, 'ckpt_format': 'torch_dist', 'ckpt_convert_format': None, 'ckpt_convert_save': None, 'ckpt_convert_update_legacy_dist_opt_format': False, 'ckpt_fully_parallel_save_deprecated': False, 'ckpt_fully_parallel_save': True, 'async_save': True, 'ckpt_fully_parallel_load': True, 'ckpt_assume_constant_structure': False, 'dist_ckpt_strictness': 'assume_ok_unexpected', 'dist_ckpt_save_pre_mcore_014': False, 'dist_ckpt_optim_fully_reshardable': False, 'distrib_optim_fully_reshardable_mem_efficient': False, 'fp16': False, 'bf16': True, 'grad_reduce_in_bf16': False, 'loss_scale': None, 'initial_loss_scale': 4294967296, 'min_loss_scale': 1.0, 'loss_scale_window': 1000, 'hysteresis': 2, 'fp32_residual_connection': False, 'apply_query_key_layer_scaling': False, 'attention_softmax_in_fp32': False, 'accumulate_allreduce_grads_in_fp32': True, 'fp16_lm_cross_entropy': False, 'disable_bf16_reduced_precision_matmul': False, 'reuse_grad_buf_for_mxfp8_param_ag': False, 'tensor_model_parallel_size': 1, 'pipeline_model_parallel_size': 1, 'decoder_first_pipeline_num_layers': None, 'decoder_last_pipeline_num_layers': None, 'pipeline_model_parallel_layout': None, 'num_layers_per_virtual_pipeline_stage': None, 'num_virtual_stages_per_pipeline_rank': None, 'microbatch_group_size_per_vp_stage': None, 'overlap_p2p_comm': False, 'overlap_p2p_comm_warmup_flush': False, 'distributed_backend': 'nccl', 'distributed_timeout_minutes': 120, 'distributed_timeout_seconds_after_init': None, 'overlap_grad_reduce': True, 'defer_grad_checks': False, 'defer_embedding_wgrad_compute': False, 'wgrad_deferral_limit': 0, 'align_grad_reduce': True, 'ddp_num_buckets': None, 'ddp_bucket_size': 40000000, 'ddp_pad_buckets_for_high_nccl_busbw': False, 'ddp_average_in_collective': False, 'overlap_param_gather': True, 'overlap_param_gather_with_optimizer_step': False, 'align_param_gather': False, 'scatter_gather_tensors_in_pipeline': True, 'use_ring_exchange_p2p': False, 'local_rank': 7, 'lazy_mpu_init': None, 'account_for_embedding_in_pipeline_split': False, 'account_for_loss_in_pipeline_split': False, 'use_distributed_optimizer': True, 'nccl_ub': False, 'disable_symmetric_registration': False, 'use_sharp': False, 'sharp_enabled_group': None, 'use_megatron_fsdp': False, 'init_model_with_meta_device': False, 'data_parallel_sharding_strategy': 'no_shard', 'gradient_reduce_div_fusion': True, 'fsdp_double_buffer': False, 'suggested_communication_unit_size': None, 'keep_fp8_transpose_cache': False, 'enable_full_sharding_in_hsdp': False, 'num_distributed_optimizer_instances': 1, 'use_torch_fsdp2': False, 'torch_fsdp2_reshard_after_forward': True, 'context_parallel_size': 8, 'cp_comm_type': ['a2a'], 'hierarchical_context_parallel_sizes': None, 'max_seqlen_per_cp_rank': None, 'hybrid_context_parallel': False, 'dynamic_context_parallel': False, 'dynamic_context_parallel_sizes': None, 'dynamic_cp_candidate_sizes': None, 'sequence_packing_scheduler': 'default_dynamic_cp', 'dynamic_cp_scheduler': 'default_dynamic_cp', 'dynamic_cp_moe_policy': None, 'dynamic_cp_gdn_policy': None, 'dynamic_cp_on_error': 'error', 'dynamic_cp_workspace_max_tokens': None, 'dynamic_cp_wave_coexecution': False, 'nccl_communicator_config_path': None, 'use_tp_pp_dp_mapping': False, 'replication': False, 'replication_jump': None, 'replication_factor': 2, 'full_validation': False, 'multiple_validation_sets': False, 'eval_iters': 10, 'eval_interval': 99999999, 'test_mode': False, 'skip_train': False, 'benchmark_eval': False, 'benchmark_tasks': None, 'benchmark_interval': None, 'benchmark_sequence_length': None, 'benchmark_micro_batch': None, 'benchmark_global_batch': None, 'data_path': ['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp'], 'split': '999,1,0', 'train_data_path': None, 'valid_data_path': None, 'test_data_path': None, 'data_args_path': None, 'load_complemental_dataset': ['loss_mask'], 'per_split_data_args_path': None, 'dataloader_fast_cache_load': False, 'dataloader_defer_npy_index_mmap': False, 'data_cache_path': '/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2', 'mmap_bin_files': True, 'mock_data': False, 'seq_length': 131072, 'encoder_seq_length': 131072, 'decoder_seq_length': None, 'retriever_seq_length': 256, 'sample_rate': 1.0, 'mask_prob': 0.15, 'short_seq_prob': 0.1, 'num_workers': 4, 'prefetch_factor': 8, 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'create_attention_mask_in_dataloader': False, 'disable_explicit_attention_mask': False, 'num_dataset_builder_threads': 1, 'object_storage_cache_path': None, 'mid_level_dataset_surplus': 0.005, 'document_packing_algorithm': 'obfd', 'document_packing_dp_size': None, 'packing_tokens_per_shard': 100000000000, 'start_samples': 0, 'consumed_train_samples': 0, 'previous_stage_seq_length': None, 'allow_ambiguous_pad_tokens': False, 'auto_generate_cu_seqlens': True, 'vocab_size': None, 'padded_vocab_size': 99072, 'vocab_file': None, 'merge_file': None, 'vocab_extra_ids': 0, 'tokenizer_type': 'HuggingFaceTokenizer', 'tokenizer_model': '/mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp', 'tokenizer_metadata': None, 'tiktoken_pattern': None, 'tiktoken_num_special_tokens': 1000, 'tiktoken_special_tokens': None, 'legacy_tokenizer': False, 'trust_remote_code': False, 'override_hf_eod_token_id': None, 'adlr_autoresume': False, 'adlr_autoresume_interval': 1000, 'ict_head_size': None, 'biencoder_projection_dim': 0, 'biencoder_shared_query_context_model': False, 'ict_load': None, 'bert_load': None, 'titles_data_path': None, 'query_in_block_prob': 0.1, 'use_one_sent_docs': False, 'evidence_data_path': None, 'retriever_report_topk_accuracies': [], 'retriever_score_scaling': False, 'block_data_path': None, 'embedding_path': None, 'indexer_batch_size': 128, 'indexer_log_interval': 1000, 'num_classes': 1000, 'img_h': 224, 'img_w': 224, 'num_channels': 3, 'patch_dim': 16, 'classes_fraction': 1.0, 'data_per_class_fraction': 1.0, 'data_sharding': True, 'head_lr_mult': 1.0, 'vision_pretraining': False, 'vision_pretraining_type': 'classify', 'vision_backbone_type': 'vit', 'swin_backbone_type': 'tiny', 'mask_type': 'random', 'mask_factor': 1.0, 'iter_per_epoch': 1250, 'dino_local_img_size': 96, 'dino_local_crops_number': 10, 'dino_head_hidden_size': 2048, 'dino_bottleneck_size': 256, 'dino_freeze_last_layer': 1, 'dino_norm_last_layer': False, 'dino_warmup_teacher_temp': 0.04, 'dino_teacher_temp': 0.07, 'dino_warmup_teacher_temp_epochs': 30, 'qk_layernorm': False, 'qk_l2_norm': False, 'expert_model_parallel_size': 8, 'expert_tensor_parallel_size': 1, 'num_experts': 192, 'moe_layer_freq': [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'moe_ffn_hidden_size': 512, 'moe_shared_expert_intermediate_size': 512, 'moe_shared_expert_gate': True, 'moe_shared_expert_overlap': True, 'moe_triton_shared_expert_overlap': False, 'moe_grouped_gemm': True, 'moe_use_legacy_grouped_gemm': False, 'moe_layer_recompute': False, 'moe_extended_tp': False, 'moe_use_upcycling': False, 'moe_router_load_balancing_type': 'global_aux_loss', 'moe_router_dtype': 'fp32', 'moe_router_gemm_backend': 'te', 'moe_router_fusion': True, 'moe_router_aux_loss_fusion': 'auto', 'moe_gaux_overlap': False, 'moe_router_score_function': 'softmax', 'moe_router_sqrt_gate': True, 'moe_router_topk': 4, 'moe_router_pre_softmax': False, 'moe_router_num_groups': None, 'moe_router_group_topk': None, 'moe_router_topk_scaling_factor': None, 'moe_router_enable_expert_bias': False, 'moe_router_bias_update_rate': 0.001, 'moe_router_force_load_balancing': False, 'moe_router_padding_for_quantization': False, 'moe_router_bias_update_method': 'sign', 'moe_router_padding_for_fp8': False, 'moe_aux_loss_coeff': 0.05, 'moe_z_loss_coeff': None, 'moe_input_jitter_eps': None, 'moe_per_layer_logging': False, 'log_moe_activations': True, 'log_moe_params': False, 'log_moe_router_metrics': False, 'moe_monitor_layers': '', 'moe_monitor_log_interval': None, 'moe_token_dispatcher_type': 'alltoall', 'moe_triton_dispatcher_exclude_padding_tokens': False, 'moe_triton_workspace_expand': 4.0, 'moe_triton_mega_dispatch_sms': 48, 'moe_triton_mega_tail_sms': 16, 'moe_triton_mega_compute_sms': 24, 'moe_triton_mega_activation_sms': 4, 'moe_triton_mega_combine_sms': 32, 'moe_triton_mega_reduce_sms': 32, 'moe_triton_mega_num_warps': 16, 'moe_enable_deepep': False, 'moe_flex_dispatcher_backend': 'deepep', 'moe_deepep_num_sms': 20, 'moe_hybridep_num_sms': 16, 'moe_permute_fusion': True, 'moe_permute_backend': 'auto', 'moe_expert_capacity_factor': None, 'moe_pad_expert_input_to_capacity': False, 'moe_token_drop_policy': 'probs', 'moe_apply_probs_on_input': False, 'overlap_moe_expert_parallel_comm': False, 'delay_wgrad_compute': False, 'moe_upcycling_granularity': 1, 'moe_pad_experts_for_cuda_graph_inference': False, 'q_lora_rank': None, 'kv_lora_rank': 32, 'qk_head_dim': 128, 'qk_pos_emb_head_dim': 64, 'v_head_dim': 128, 'rotary_scaling_factor': 1.0, 'mscale': 1.0, 'mscale_all_dim': 0.0, 'cache_mla_latents': False, 'linear_attention_type': 'gated_delta_net', 'linear_attention_freq': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1], 'linear_rope_mode': 'none', 'dc_shortconv': False, 'dc_shortconv_upcycling': False, 'dc_shortconv_optimizer_state': 'conservative', 'gdn_shortconv_separate_gate': False, 'gdn_shortconv_separate_gate_upcycling': False, 'linear_conv_kernel_dim': 4, 'linear_key_head_dim': 64, 'linear_value_head_dim': 64, 'linear_num_key_heads': 8, 'linear_num_value_heads': 32, 'gdn_pre_gdr_fusion': False, 'gdn_pre_gdr_fusion_high_precision_backward': False, 'gdn_pre_gdr_fusion_deterministic_weight_gradient': False, 'gdn_backend': 'fla', 'gdn_gated_norm_backend': 'linghe', 'gdn_split_in_proj': True, 'gdn_fused_split_forward': True, 'gdn_postinit_key_scale': 1.0, 'gdn_postinit_alpha_scale': 1.0, 'gdn_postinit_alog_shift': 0.0, 'gdn_alpha_weight_scale': 1.0, 'gdn_alpha_weight_scale_mode': 'matrix', 'gdn_lr_mult_q': 1.0, 'gdn_lr_mult_k': 1.0, 'gdn_lr_mult_v': 1.0, 'gdn_lr_mult_alpha': 0.5, 'gdn_lr_mult_beta': 1.0, 'heterogeneous_layers_config_path': None, 'heterogeneous_layers_config_encoded_json': None, 'log_params_norm': True, 'log_padding_stats': False, 'log_num_zeros_in_grad': False, 'log_throughput': True, 'log_progress': False, 'timing_log_level': 0, 'log_energy': False, 'increase_log_level_interval': 1000, 'increase_log_level_iters': 5, 'barrier_with_L1_time': False, 'timing_log_option': 'all', 'tensorboard_log_interval': 1, 'tensorboard_queue_size': 1000, 'log_timers_to_tensorboard': False, 'log_loss_scale_to_tensorboard': True, 'log_validation_ppl_to_tensorboard': False, 'log_memory_to_tensorboard': False, 'log_world_size_to_tensorboard': False, 'wandb_project': 'qwen2.5_3b_mom_cpt', 'wandb_entity': '', 'wandb_exp_name': 'stage14_120B_8node_gdnlralpha_off_20260925a', 'wandb_save_dir': '/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb', 'swanlab_project': '', 'swanlab_workspace': '', 'swanlab_exp_name': '', 'swanlab_save_dir': '', 'logging_level': None, 'enable_debug_logging': False, 'log_hidden_states': ['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'], 'log_params': ['attention::linear_qkv', 'attention::o_proj', 'mlp::linear_fc1', 'mlp::linear_fc2', 'input_layernorm', 'pre_mlp_layernorm', 'embedding', 'lm_head', 'gated_delta::in_proj', 'gated_delta::in_proj_bias', 'gated_delta::conv1d', 'gated_delta::conv1d_bias', 'gated_delta::dt_bias', 'gated_delta::A_log', 'gated_delta::out_norm', 'gated_delta::out_norm_bias', 'gated_delta::out_proj', 'gated_delta::out_proj_bias'], 'log_attn_logits': False, 'log_logits_z_loss': False, 'log_metrics_interval': 10, 'log_update_rms': True, 'output_logits_z_loss_coeff': None, 'log_straggler': False, 'disable_straggler_on_startup': False, 'straggler_ctrlr_port': 65535, 'straggler_minmax_count': 1, 'run_workload_inspector_server': False, 'inference_batch_times_seqlen_threshold': -1, 'max_tokens_to_oom': 12000, 'output_bert_embeddings': False, 'bert_embedder_type': 'megatron', 'flash_decode': False, 'enable_cuda_graph': False, 'cuda_graph_warmup_steps': 3, 'external_cuda_graph': False, 'cuda_graph_impl': 'none', 'cuda_graph_scope': [], 'inference_max_batch_size': 8, 'inference_max_seq_length': 2560, 'inference_dynamic_batching': False, 'inference_dynamic_batching_buffer_size_gb': 40.0, 'inference_dynamic_batching_block_size': 256, 'inference_dynamic_batching_buffer_guaranteed_fraction': 0.2, 'inference_dynamic_batching_buffer_overflow_factor': None, 'inference_dynamic_batching_max_requests_override': None, 'inference_dynamic_batching_max_tokens_override': None, 'inference_dynamic_batching_num_cuda_graphs': 16, 'inference_dynamic_batching_track_paused_request_events': False, 'decode_only_cuda_graphs': False, 'inference_dynamic_batching_unified_memory_level': 0, 'symmetric_ar_type': None, 'nccl_all_reduce_for_prefill': False, 'mlp_chunks_for_prefill': 1, 'disable_chunked_prefill': False, 'fp8': None, 'fp8_recipe': 'delayed', 'fp8_margin': 0, 'fp8_interval': 1, 'fp8_amax_history_len': 1, 'fp8_amax_compute_algo': 'most_recent', 'fp8_wgrad': True, 'transformer_impl': 'transformer_engine', 'fp8_param_gather': False, 'first_last_layers_bf16': False, 'num_layers_at_start_in_bf16': 1, 'num_layers_at_end_in_bf16': 1, 'fp4': None, 'fp4_recipe': 'nvfp4', 'fp4_param': False, 'te_rng_tracker': False, 'inference_rng_tracker': False, 'retro_project_dir': None, 'retro_add_retriever': False, 'retro_cyclic_train_iters': None, 'retro_encoder_layers': 2, 'retro_encoder_hidden_dropout': 0.1, 'retro_encoder_attention_dropout': 0.1, 'retro_num_neighbors': 2, 'retro_num_retrieved_chunks': 2, 'retro_attention_gate': 1, 'retro_verify_neighbor_count': True, 'enable_experimental': False, 'spec': None, 'hybrid_attention_ratio': 0.0625, 'hybrid_mlp_ratio': 0.5, 'hybrid_override_pattern': None, 'mamba_state_dim': 128, 'mamba_head_dim': 64, 'mamba_num_groups': 8, 'mamba_num_heads': None, 'mamba_expand': 2, 'is_hybrid_model': True, 'disable_mamba_mem_eff_path': False, 'mamba_disable_cp': False, 'yaml_cfg': None, 'use_precision_aware_optimizer': False, 'main_grads_dtype': torch.float32, 'main_params_dtype': torch.float32, 'exp_avg_dtype': torch.float32, 'exp_avg_sq_dtype': torch.float32, 'enable_one_logger': True, 'one_logger_project': 'megatron-lm', 'one_logger_run_name': None, 'one_logger_async': False, 'app_tag_run_name': None, 'app_tag_run_version': '0.0.0', 'inprocess_restart': False, 'inprocess_max_iterations': None, 'inprocess_monitor_thread_interval': 1.0, 'inprocess_monitor_process_interval': 1.0, 'inprocess_progress_watchdog_interval': 1.0, 'inprocess_heartbeat_interval': 30, 'inprocess_soft_timeout': 60, 'inprocess_hard_timeout': 90, 'inprocess_heartbeat_timeout': 60, 'inprocess_barrier_timeout': 120, 'inprocess_completion_timeout': 120, 'inprocess_last_call_wait': 1, 'inprocess_termination_grace_time': 1, 'inprocess_granularity': 'node', 'inprocess_active_world_size': 64, 'inprocess_empty_cuda_cache': False, 'enable_ft_package': False, 'calc_ft_timeouts': False, 'config_logger_dir': '', 'error_injection_rate': 0, 'error_injection_type': 'transient_error', 'rerun_mode': 'validate_results', 'enable_msc': True, 'kitchen_config_file': None, 'kitchen_recipe_number': None, 'sft': False, 'sft_tokenizer_prompt_format': 'nemotron-h-aligned', 'export_model_type': 'GPTModel', 'export_legacy_megatron': False, 'export_te_mcore_model': False, 'export_force_local_attention': False, 'export_kv_cache_quant': False, 'export_real_quant_cfg': 'None', 'export_quant_cfg': None, 'export_kd_cfg': None, 'teacher_model_config': None, 'export_kd_teacher_load': None, 'export_kd_teacher_ckpt_format': None, 'finetune_hf_dataset': None, 'finetune_data_split': 'train', 'export_qk_l2_norm': False, 'export_moe_apply_probs_on_input': False, 'export_offline_model': False, 'modelopt_enabled': False, 'rank': 63, 'world_size': 64, 'use_dist_ckpt': True, 'transformer_pipeline_model_parallel_size': 1, 'data_parallel_size': 8, 'virtual_pipeline_model_parallel_size': None, 'params_dtype': torch.bfloat16, 'skipped_train_samples': 0, 'consumed_valid_samples': 0, 'variable_seq_lengths': False, '_wandb': {}}
+2026-10-03 15:36:55,091 INFO    MainThread:730851 [wandb_init.py:init():888] starting backend
+2026-10-03 15:36:55,382 INFO    MainThread:730851 [wandb_init.py:init():891] sending inform_init request
+2026-10-03 15:36:55,389 INFO    MainThread:730851 [wandb_init.py:init():899] backend started and connected
+2026-10-03 15:36:55,396 INFO    MainThread:730851 [wandb_init.py:init():969] updated telemetry
+2026-10-03 15:36:55,396 INFO    MainThread:730851 [wandb_init.py:init():993] communicating run to backend with 90.0 second timeout
+2026-10-03 15:36:57,237 INFO    MainThread:730851 [wandb_init.py:init():1040] starting run threads in backend
+2026-10-03 15:36:57,542 INFO    MainThread:730851 [wandb_run.py:_console_start():2504] atexit reg
+2026-10-03 15:36:57,542 INFO    MainThread:730851 [wandb_run.py:_redirect():2352] redirect: wrap_raw
+2026-10-03 15:36:57,542 INFO    MainThread:730851 [wandb_run.py:_redirect():2421] Wrapping output streams.
+2026-10-03 15:36:57,542 INFO    MainThread:730851 [wandb_run.py:_redirect():2444] Redirects installed.
+2026-10-03 15:36:57,547 INFO    MainThread:730851 [wandb_init.py:init():1080] run started, returning control to user process
+2026-10-03 15:47:07,644 INFO    MainThread:730851 [wandb_run.py:_config_callback():1385] config_cb None None {'slurm_job_name': 'N/A'}
+2026-10-10 20:25:46,273 INFO    MainThread:730851 [wandb_run.py:_finish():2270] finishing run huyiwen/qwen2.5_3b_mom_cpt/neb76hvj
+2026-10-10 20:25:46,275 INFO    MainThread:730851 [wandb_run.py:_atexit_cleanup():2469] got exitcode: 0
+2026-10-10 20:25:46,275 INFO    MainThread:730851 [wandb_run.py:_restore():2451] restore
+2026-10-10 20:25:46,276 INFO    MainThread:730851 [wandb_run.py:_restore():2457] restore done
+2026-10-10 20:25:53,585 INFO    MainThread:730851 [wandb_run.py:_footer_sync_info():3853] logging synced files

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__18_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:449d880a14c43ba7b4a7879dabf7504c9fccb1888ff98c46b9135193ac15151c
+size 3725451014

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__21_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9321c3c0c304b3d47b9834e9fd260c40a5b3b3efa796a760763529d841456a9e
+size 3725472260

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__29_1.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adc9883d8aad9a4e90c3631842c433f02e0227a7e5a4360e1e442dfecdcb64a6
+size 3725405514

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__49_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7385d4c3dd6444331feb90763b617dd318b7135dd44b29429d4ed504d28e65fc
+size 3725314012

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/metadata.json

@@ -0,0 +1 @@
+{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1}

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/modelopt_run_config.yaml

@@ -0,0 +1,833 @@
+activation_func: <function silu at 0x7fb16609bd00>
+activation_func_clamp_mode: soft
+activation_func_clamp_value: '7.0'
+add_bias_linear: false
+add_qkv_bias: true
+apply_query_key_layer_scaling: false
+apply_residual_connection_post_layernorm: false
+apply_rope_fusion: false
+attention_backend: AttnBackend.auto
+attention_dropout: '0.0'
+attention_output_gate: true
+attention_output_gate_upcycling: false
+attention_softmax_in_fp32: false
+attn_k_token_shift: None
+attn_output_gate: None
+attn_q_token_shift: None
+attn_token_shift: None
+attn_v_token_shift: None
+autocast_dtype: torch.bfloat16
+barrier_with_L1_time: false
+bf16: true
+bias_activation_fusion: true
+bias_dropout_fusion: true
+calculate_per_token_loss: false
+clone_scatter_output_in_embedding: true
+config_logger_dir: ''
+cross_entropy_fusion_impl: linear_triton
+cross_entropy_loss_fusion: true
+dc_shortconv: false
+dc_shortconv_upcycling: false
+defer_embedding_wgrad_compute: false
+delay_wgrad_compute: false
+deterministic_mode: false
+disable_bf16_reduced_precision_matmul: false
+disable_parameter_transpose_cache: false
+distribute_saved_activations: false
+dynamic_cp_candidate_sizes: None
+dynamic_cp_gdn_policy: None
+dynamic_cp_moe_policy: None
+dynamic_cp_on_error: error
+dynamic_cp_scheduler: default_dynamic_cp
+dynamic_cp_wave_coexecution: false
+dynamic_cp_workspace_max_tokens: None
+emb_deviation_loss_coeff: 0
+emb_deviation_type: None
+enable_autocast: false
+fast_thd_a2a: true
+ffn_hidden_size: 4800
+ffn_intermediate_token_shift: None
+ffn_token_shift: None
+final_logits_clamp_value: '30.0'
+finalize_model_grads_func: <function finalize_model_grads at 0x7fb0796a6200>
+fine_grained_activation_offloading: false
+first_last_layers_bf16: false
+flash_decode: false
+fp16: false
+fp32_residual_connection: false
+freeze_layernorm_weight: false
+fused_single_qkv_rope: false
+gated_linear_unit: true
+gated_norm: true
+gated_norm_lora_rank: 16
+gated_norm_upcycling: false
+gdn_alpha_weight_scale: '1.0'
+gdn_alpha_weight_scale_mode: matrix
+gdn_backend: fla
+gdn_fused_split_forward: true
+gdn_gated_norm_backend: linghe
+gdn_lr_mult_alpha: '1.0'
+gdn_lr_mult_beta: '1.0'
+gdn_lr_mult_k: '1.0'
+gdn_lr_mult_q: '1.0'
+gdn_lr_mult_v: '1.0'
+gdn_pre_gdr_fusion: false
+gdn_pre_gdr_fusion_deterministic_weight_gradient: false
+gdn_pre_gdr_fusion_high_precision_backward: false
+gdn_shortconv_separate_gate: false
+gdn_shortconv_separate_gate_upcycling: false
+gdn_split_in_proj: true
+glu_linear_offset: '0.0'
+grad_scale_func: <bound method MegatronOptimizer.scale_loss of <megatron.core.optimizer.optimizer.ChainedOptimizer
+  object at 0x7fafb0138e50>>
+grad_sync_func: "<bound method DistributedDataParallel.start_grad_sync of DistributedDataParallel(\n\
+  \  (module): Float16Module(\n    (module): GPTModel(\n      (embedding): LanguageModelEmbedding(\n\
+  \        (word_embeddings): VocabParallelEmbedding()\n        (embedding_dropout):\
+  \ Dropout(p=0.0, inplace=False)\n      )\n      (decoder): TransformerBlock(\n \
+  \       (layers): ModuleList(\n          (0-1): 2 x TransformerLayer(\n        \
+  \    (input_layernorm): GatedNorm(\n              (norm): RMSNorm()\n          \
+  \    (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n      \
+  \        (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n     \
+  \       )\n            (self_attention): GatedDeltaNet(\n              (in_proj_layer_norm):\
+  \ IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MLP(\n              (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=9600, bias=False, TP=1)\n              (linear_fc2): TERowParallelLinear(in_features=4800,\
+  \ out_features=2048, bias=False, TP=1)\n            )\n          )\n          (2-11):\
+  \ 10 x TransformerLayer(\n            (input_layernorm): GatedNorm(\n          \
+  \    (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048, out_features=16,\
+  \ bias=False)\n              (gate_up): Linear(in_features=16, out_features=2048,\
+  \ bias=True)\n            )\n            (self_attention): GatedDeltaNet(\n    \
+  \          (in_proj_layer_norm): IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (12): TransformerLayer(\n            (input_layernorm): GatedNorm(\n\
+  \              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): SelfAttention(\n\
+  \              (core_attention): TEDotProductAttention(\n                (flash_attention):\
+  \ FlashAttention()\n                (fused_attention): FusedAttention()\n      \
+  \          (unfused_attention): UnfusedDotProductAttention(\n                  (scale_mask_softmax):\
+  \ FusedScaleMaskSoftmax()\n                  (attention_dropout): Dropout(p=0.0,\
+  \ inplace=False)\n                )\n              )\n              (linear_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \             (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120,\
+  \ bias=True, TP=1)\n              (q_layernorm): IdentityOp()\n              (k_layernorm):\
+  \ IdentityOp()\n            )\n            (pre_cross_attn_layernorm): IdentityOp()\n\
+  \            (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (13-19): 7 x TransformerLayer(\n            (input_layernorm):\
+  \ GatedNorm(\n              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): GatedDeltaNet(\n\
+  \              (in_proj_layer_norm): IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (20-21): 2 x TransformerLayer(\n            (input_layernorm):\
+  \ GatedNorm(\n              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): SelfAttention(\n\
+  \              (core_attention): TEDotProductAttention(\n                (flash_attention):\
+  \ FlashAttention()\n                (fused_attention): FusedAttention()\n      \
+  \          (unfused_attention): UnfusedDotProductAttention(\n                  (scale_mask_softmax):\
+  \ FusedScaleMaskSoftmax()\n                  (attention_dropout): Dropout(p=0.0,\
+  \ inplace=False)\n                )\n              )\n              (linear_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \             (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120,\
+  \ bias=True, TP=1)\n              (q_layernorm): IdentityOp()\n              (k_layernorm):\
+  \ IdentityOp()\n            )\n            (pre_cross_attn_layernorm): IdentityOp()\n\
+  \            (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (22): TransformerLayer(\n            (input_layernorm): GatedNorm(\n\
+  \              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): GatedDeltaNet(\n\
+  \              (in_proj_layer_norm): IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (23): TransformerLayer(\n            (input_layernorm): GatedNorm(\n\
+  \              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): SelfAttention(\n\
+  \              (core_attention): TEDotProductAttention(\n                (flash_attention):\
+  \ FlashAttention()\n                (fused_attention): FusedAttention()\n      \
+  \          (unfused_attention): UnfusedDotProductAttention(\n                  (scale_mask_softmax):\
+  \ FusedScaleMaskSoftmax()\n                  (attention_dropout): Dropout(p=0.0,\
+  \ inplace=False)\n                )\n              )\n              (linear_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \             (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120,\
+  \ bias=True, TP=1)\n              (q_layernorm): IdentityOp()\n              (k_layernorm):\
+  \ IdentityOp()\n            )\n            (pre_cross_attn_layernorm): IdentityOp()\n\
+  \            (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (24-45): 22 x TransformerLayer(\n            (input_layernorm):\
+  \ GatedNorm(\n              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): GatedDeltaNet(\n\
+  \              (in_proj_layer_norm): IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (46): TransformerLayer(\n            (input_layernorm): GatedNorm(\n\
+  \              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): SelfAttention(\n\
+  \              (core_attention): TEDotProductAttention(\n                (flash_attention):\
+  \ FlashAttention()\n                (fused_attention): FusedAttention()\n      \
+  \          (unfused_attention): UnfusedDotProductAttention(\n                  (scale_mask_softmax):\
+  \ FusedScaleMaskSoftmax()\n                  (attention_dropout): Dropout(p=0.0,\
+  \ inplace=False)\n                )\n              )\n              (linear_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \             (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120,\
+  \ bias=True, TP=1)\n              (q_layernorm): IdentityOp()\n              (k_layernorm):\
+  \ IdentityOp()\n            )\n            (pre_cross_attn_layernorm): IdentityOp()\n\
+  \            (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (47): TransformerLayer(\n            (input_layernorm): GatedNorm(\n\
+  \              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): GatedDeltaNet(\n\
+  \              (in_proj_layer_norm): IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (48-49): 2 x TransformerLayer(\n            (input_layernorm):\
+  \ GatedNorm(\n              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): SelfAttention(\n\
+  \              (core_attention): TEDotProductAttention(\n                (flash_attention):\
+  \ FlashAttention()\n                (fused_attention): FusedAttention()\n      \
+  \          (unfused_attention): UnfusedDotProductAttention(\n                  (scale_mask_softmax):\
+  \ FusedScaleMaskSoftmax()\n                  (attention_dropout): Dropout(p=0.0,\
+  \ inplace=False)\n                )\n              )\n              (linear_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \             (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120,\
+  \ bias=True, TP=1)\n              (q_layernorm): IdentityOp()\n              (k_layernorm):\
+  \ IdentityOp()\n            )\n            (pre_cross_attn_layernorm): IdentityOp()\n\
+  \            (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (50-55): 6 x TransformerLayer(\n            (input_layernorm):\
+  \ GatedNorm(\n              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): GatedDeltaNet(\n\
+  \              (in_proj_layer_norm): IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n        )\n        (final_layernorm): GatedNorm(\n          (norm): RMSNorm()\n\
+  \          (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \          (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n   \
+  \     )\n      )\n      (output_layer): FusedLinearCrossEntropyColumnParallelLinear(in_features=2048,\
+  \ out_features=99072, bias=False, TP=1)\n    )\n  )\n)>"
+gradient_accumulation_fusion: true
+hetereogenous_dist_checkpoint: false
+heterogeneous_block_specs: false
+hidden_dropout: '0.0'
+hidden_size: 2048
+hidden_states_shrink: '1.0'
+is_hybrid_model: true
+kv_channels: 64
+layernorm_epsilon: 1e-06
+layernorm_zero_centered_gamma: false
+linear_attention_freq: '[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1,
+  1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  0, 1, 0, 0, 1, 1, 1, 1, 1, 1]'
+linear_attention_type: gated_delta_net
+linear_conv_kernel_dim: 4
+linear_cross_entropy_accumulate_grad_in_fp32: true
+linear_cross_entropy_num_chunks: 32
+linear_key_head_dim: 64
+linear_num_key_heads: 8
+linear_num_value_heads: 32
+linear_rope_mode: none
+linear_value_head_dim: 64
+lm_head_alpha: '1.0'
+log_attn_logits: false
+log_hidden_states: '[''embeddings'', ''residual_attn'', ''input_layernorm'', ''attention::q_layernorm'',
+  ''attention::k_layernorm'', ''attention::core_attention'', ''attention::o_proj'',
+  ''residual_mlp'', ''pre_mlp_layernorm'', ''mlp'', ''moe::router'', ''moe::shared_experts'',
+  ''moe::routed_experts'', ''gated_delta::in_proj'', ''gated_delta::conv1d'', ''gated_delta::query'',
+  ''gated_delta::key'', ''gated_delta::value'', ''gated_delta::gate'', ''gated_delta::beta'',
+  ''gated_delta::alpha'', ''gated_delta::g'', ''gated_delta::core_attention'', ''gated_delta::out_norm'',
+  ''gated_delta::out_proj'', ''final_layernorm'']'
+log_logits_z_loss: false
+log_metrics_interval: 10
+log_moe_activations: true
+log_moe_params: false
+log_moe_router_metrics: false
+log_params: '[''attention::linear_qkv'', ''attention::o_proj'', ''mlp::linear_fc1'',
+  ''mlp::linear_fc2'', ''input_layernorm'', ''pre_mlp_layernorm'', ''embedding'',
+  ''lm_head'', ''gated_delta::in_proj'', ''gated_delta::in_proj_bias'', ''gated_delta::conv1d'',
+  ''gated_delta::conv1d_bias'', ''gated_delta::dt_bias'', ''gated_delta::A_log'',
+  ''gated_delta::out_norm'', ''gated_delta::out_norm_bias'', ''gated_delta::out_proj'',
+  ''gated_delta::out_proj_bias'']'
+log_update_rms: true
+mamba_disable_cp: false
+mamba_expand: 2
+mamba_head_dim: 64
+mamba_num_groups: 8
+mamba_num_heads: None
+mamba_state_dim: 128
+masked_softmax_fusion: false
+max_seqlen_per_dp_cp_rank: None
+memory_efficient_layer_norm: false
+min_offloaded_tensor_size: 1048576
+mlp_chunks_for_prefill: 1
+moe_apply_probs_on_input: false
+moe_aux_loss_coeff: '0.05'
+moe_deepep_num_sms: 20
+moe_enable_deepep: false
+moe_expert_capacity_factor: None
+moe_extended_tp: false
+moe_ffn_hidden_size: 512
+moe_flex_dispatcher_backend: deepep
+moe_gaux_overlap: false
+moe_grouped_gemm: true
+moe_hybridep_num_sms: 16
+moe_input_jitter_eps: None
+moe_layer_freq: '[0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1]'
+moe_pad_expert_input_to_capacity: false
+moe_per_layer_logging: false
+moe_permute_backend: auto
+moe_permute_fusion: true
+moe_router_aux_loss_fusion: auto
+moe_router_bias_update_method: sign
+moe_router_bias_update_rate: '0.001'
+moe_router_dtype: fp32
+moe_router_enable_expert_bias: false
+moe_router_force_load_balancing: false
+moe_router_fusion: true
+moe_router_gemm_backend: te
+moe_router_group_topk: None
+moe_router_load_balancing_type: global_aux_loss
+moe_router_num_groups: None
+moe_router_padding_for_quantization: false
+moe_router_pre_softmax: false
+moe_router_score_function: softmax
+moe_router_sqrt_gate: true
+moe_router_topk: 4
+moe_router_topk_limited_devices: None
+moe_router_topk_scaling_factor: None
+moe_shared_expert_gate: true
+moe_shared_expert_intermediate_size: 512
+moe_shared_expert_overlap: true
+moe_token_dispatcher_type: alltoall
+moe_token_drop_policy: probs
+moe_token_dropping: false
+moe_triton_dispatcher_exclude_padding_tokens: false
+moe_triton_mega_activation_sms: 4
+moe_triton_mega_combine_sms: 32
+moe_triton_mega_compute_sms: 24
+moe_triton_mega_dispatch_sms: 48
+moe_triton_mega_num_warps: 16
+moe_triton_mega_reduce_sms: 32
+moe_triton_mega_tail_sms: 16
+moe_triton_shared_expert_overlap: false
+moe_triton_workspace_expand: '4.0'
+moe_triton_workspace_max_tokens: None
+moe_use_legacy_grouped_gemm: false
+moe_z_loss_coeff: None
+mrope_section: None
+multi_latent_attention: false
+no_rope_freq: None
+no_sync_func: "<bound method DistributedDataParallel.no_sync of DistributedDataParallel(\n\
+  \  (module): Float16Module(\n    (module): GPTModel(\n      (embedding): LanguageModelEmbedding(\n\
+  \        (word_embeddings): VocabParallelEmbedding()\n        (embedding_dropout):\
+  \ Dropout(p=0.0, inplace=False)\n      )\n      (decoder): TransformerBlock(\n \
+  \       (layers): ModuleList(\n          (0-1): 2 x TransformerLayer(\n        \
+  \    (input_layernorm): GatedNorm(\n              (norm): RMSNorm()\n          \
+  \    (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n      \
+  \        (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n     \
+  \       )\n            (self_attention): GatedDeltaNet(\n              (in_proj_layer_norm):\
+  \ IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MLP(\n              (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=9600, bias=False, TP=1)\n              (linear_fc2): TERowParallelLinear(in_features=4800,\
+  \ out_features=2048, bias=False, TP=1)\n            )\n          )\n          (2-11):\
+  \ 10 x TransformerLayer(\n            (input_layernorm): GatedNorm(\n          \
+  \    (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048, out_features=16,\
+  \ bias=False)\n              (gate_up): Linear(in_features=16, out_features=2048,\
+  \ bias=True)\n            )\n            (self_attention): GatedDeltaNet(\n    \
+  \          (in_proj_layer_norm): IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (12): TransformerLayer(\n            (input_layernorm): GatedNorm(\n\
+  \              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): SelfAttention(\n\
+  \              (core_attention): TEDotProductAttention(\n                (flash_attention):\
+  \ FlashAttention()\n                (fused_attention): FusedAttention()\n      \
+  \          (unfused_attention): UnfusedDotProductAttention(\n                  (scale_mask_softmax):\
+  \ FusedScaleMaskSoftmax()\n                  (attention_dropout): Dropout(p=0.0,\
+  \ inplace=False)\n                )\n              )\n              (linear_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \             (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120,\
+  \ bias=True, TP=1)\n              (q_layernorm): IdentityOp()\n              (k_layernorm):\
+  \ IdentityOp()\n            )\n            (pre_cross_attn_layernorm): IdentityOp()\n\
+  \            (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (13-19): 7 x TransformerLayer(\n            (input_layernorm):\
+  \ GatedNorm(\n              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): GatedDeltaNet(\n\
+  \              (in_proj_layer_norm): IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (20-21): 2 x TransformerLayer(\n            (input_layernorm):\
+  \ GatedNorm(\n              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): SelfAttention(\n\
+  \              (core_attention): TEDotProductAttention(\n                (flash_attention):\
+  \ FlashAttention()\n                (fused_attention): FusedAttention()\n      \
+  \          (unfused_attention): UnfusedDotProductAttention(\n                  (scale_mask_softmax):\
+  \ FusedScaleMaskSoftmax()\n                  (attention_dropout): Dropout(p=0.0,\
+  \ inplace=False)\n                )\n              )\n              (linear_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \             (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120,\
+  \ bias=True, TP=1)\n              (q_layernorm): IdentityOp()\n              (k_layernorm):\
+  \ IdentityOp()\n            )\n            (pre_cross_attn_layernorm): IdentityOp()\n\
+  \            (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (22): TransformerLayer(\n            (input_layernorm): GatedNorm(\n\
+  \              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): GatedDeltaNet(\n\
+  \              (in_proj_layer_norm): IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (23): TransformerLayer(\n            (input_layernorm): GatedNorm(\n\
+  \              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): SelfAttention(\n\
+  \              (core_attention): TEDotProductAttention(\n                (flash_attention):\
+  \ FlashAttention()\n                (fused_attention): FusedAttention()\n      \
+  \          (unfused_attention): UnfusedDotProductAttention(\n                  (scale_mask_softmax):\
+  \ FusedScaleMaskSoftmax()\n                  (attention_dropout): Dropout(p=0.0,\
+  \ inplace=False)\n                )\n              )\n              (linear_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \             (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120,\
+  \ bias=True, TP=1)\n              (q_layernorm): IdentityOp()\n              (k_layernorm):\
+  \ IdentityOp()\n            )\n            (pre_cross_attn_layernorm): IdentityOp()\n\
+  \            (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (24-45): 22 x TransformerLayer(\n            (input_layernorm):\
+  \ GatedNorm(\n              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): GatedDeltaNet(\n\
+  \              (in_proj_layer_norm): IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (46): TransformerLayer(\n            (input_layernorm): GatedNorm(\n\
+  \              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): SelfAttention(\n\
+  \              (core_attention): TEDotProductAttention(\n                (flash_attention):\
+  \ FlashAttention()\n                (fused_attention): FusedAttention()\n      \
+  \          (unfused_attention): UnfusedDotProductAttention(\n                  (scale_mask_softmax):\
+  \ FusedScaleMaskSoftmax()\n                  (attention_dropout): Dropout(p=0.0,\
+  \ inplace=False)\n                )\n              )\n              (linear_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \             (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120,\
+  \ bias=True, TP=1)\n              (q_layernorm): IdentityOp()\n              (k_layernorm):\
+  \ IdentityOp()\n            )\n            (pre_cross_attn_layernorm): IdentityOp()\n\
+  \            (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (47): TransformerLayer(\n            (input_layernorm): GatedNorm(\n\
+  \              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): GatedDeltaNet(\n\
+  \              (in_proj_layer_norm): IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (48-49): 2 x TransformerLayer(\n            (input_layernorm):\
+  \ GatedNorm(\n              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): SelfAttention(\n\
+  \              (core_attention): TEDotProductAttention(\n                (flash_attention):\
+  \ FlashAttention()\n                (fused_attention): FusedAttention()\n      \
+  \          (unfused_attention): UnfusedDotProductAttention(\n                  (scale_mask_softmax):\
+  \ FusedScaleMaskSoftmax()\n                  (attention_dropout): Dropout(p=0.0,\
+  \ inplace=False)\n                )\n              )\n              (linear_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \             (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120,\
+  \ bias=True, TP=1)\n              (q_layernorm): IdentityOp()\n              (k_layernorm):\
+  \ IdentityOp()\n            )\n            (pre_cross_attn_layernorm): IdentityOp()\n\
+  \            (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n          (50-55): 6 x TransformerLayer(\n            (input_layernorm):\
+  \ GatedNorm(\n              (norm): RMSNorm()\n              (gate_down): Linear(in_features=2048,\
+  \ out_features=16, bias=False)\n              (gate_up): Linear(in_features=16,\
+  \ out_features=2048, bias=True)\n            )\n            (self_attention): GatedDeltaNet(\n\
+  \              (in_proj_layer_norm): IdentityOp()\n              (q_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (k_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=512, bias=False, TP=1)\n              (v_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (z_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=2048, bias=False, TP=1)\n              (b_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (a_proj): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=32, bias=False, TP=1)\n              (_fused_split_linear): _FusedSplitLinearBoundary()\n\
+  \              (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,),\
+  \ groups=3072, bias=False)\n              (out_norm): RMSNorm()\n              (out_proj):\
+  \ TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1)\n \
+  \           )\n            (pre_cross_attn_layernorm): IdentityOp()\n          \
+  \  (cross_attention): IdentityOp()\n            (cross_attn_bda): IdentityFuncOp()\n\
+  \            (pre_mlp_layernorm): GatedNorm(\n              (norm): RMSNorm()\n\
+  \              (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \              (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n\
+  \            )\n            (mlp): MoELayer(\n              (router): TopKRouter(topk=4,\
+  \ routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False,\
+  \ fused=True)\n              (experts): A100GroupedMLP()\n              (shared_experts):\
+  \ SharedExpertMLP(\n                (linear_fc1): TEColumnParallelLinear(in_features=2048,\
+  \ out_features=1024, bias=False, TP=1)\n                (linear_fc2): TERowParallelLinear(in_features=512,\
+  \ out_features=2048, bias=False, TP=1)\n              )\n            )\n       \
+  \   )\n        )\n        (final_layernorm): GatedNorm(\n          (norm): RMSNorm()\n\
+  \          (gate_down): Linear(in_features=2048, out_features=16, bias=False)\n\
+  \          (gate_up): Linear(in_features=16, out_features=2048, bias=True)\n   \
+  \     )\n      )\n      (output_layer): FusedLinearCrossEntropyColumnParallelLinear(in_features=2048,\
+  \ out_features=99072, bias=False, TP=1)\n    )\n  )\n)>"
+normalization: RMSNorm
+num_attention_heads: 32
+num_layers: 56
+num_layers_at_end_in_bf16: 1
+num_layers_at_start_in_bf16: 1
+num_moe_experts: 192
+num_query_groups: 8
+nvidia_modelopt_version: 0.39.0
+offload_modules: '[]'
+output_logits_z_loss_coeff: None
+param_sync_func: None
+params_dtype: torch.bfloat16
+perform_initialization: false
+persist_layer_norm: true
+qk_l2_norm: false
+qk_layernorm: false
+quant_recipe: None
+reparam_keys: None
+rotary_interleaved: false
+scale_emb: '1.0'
+sequence_packing_scheduler: default_dynamic_cp
+softmax_scale: None
+softmax_type: vanilla
+spectral_mup_init: false
+split_expert_init: true
+split_fc1_init: true
+split_qkv_init: true
+symmetric_ar_type: None
+test_mode: false
+timers: <megatron.core.timers.Timers object at 0x7faff3f5e290>
+token_shift_conv_init: default
+token_shift_conv_size: 4
+transformer_impl: transformer_engine
+use_fused_weighted_squared_relu: false
+use_kitchen: false
+use_mamba_mem_eff_path: true
+use_ring_exchange_p2p: false
+use_te_activation_func: false
+use_te_rng_tracker: false
+variable_seq_lengths: false
+wgrad_deferral_limit: 0
+window_attn_skip_freq: None
+window_size: None
+word_embedding_dropout_prob: '0.0'

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/latest_checkpointed_iteration.txt

@@ -0,0 +1 @@
+2384

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/latest_wandb_artifact_path.txt

@@ -0,0 +1 @@
+huyiwen/qwen2.5_3b_mom_cpt

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0009.json

@@ -0,0 +1,84 @@
+{
+  "rank": 9,
+  "world_diagnostics_gloo": true,
+  "aliases": [
+    [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15,
+      16,
+      17,
+      18,
+      19,
+      20,
+      21,
+      22,
+      23,
+      24,
+      25,
+      26,
+      27,
+      28,
+      29,
+      30,
+      31,
+      32,
+      33,
+      34,
+      35,
+      36,
+      37,
+      38,
+      39,
+      40,
+      41,
+      42,
+      43,
+      44,
+      45,
+      46,
+      47,
+      48,
+      49,
+      50,
+      51,
+      52,
+      53,
+      54,
+      55,
+      56,
+      57,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63
+    ],
+    [
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15
+    ]
+  ],
+  "synchronous_singletons": true,
+  "async_work_handles_preserved": true
+}

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0014.json

@@ -0,0 +1,84 @@
+{
+  "rank": 14,
+  "world_diagnostics_gloo": true,
+  "aliases": [
+    [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15,
+      16,
+      17,
+      18,
+      19,
+      20,
+      21,
+      22,
+      23,
+      24,
+      25,
+      26,
+      27,
+      28,
+      29,
+      30,
+      31,
+      32,
+      33,
+      34,
+      35,
+      36,
+      37,
+      38,
+      39,
+      40,
+      41,
+      42,
+      43,
+      44,
+      45,
+      46,
+      47,
+      48,
+      49,
+      50,
+      51,
+      52,
+      53,
+      54,
+      55,
+      56,
+      57,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63
+    ],
+    [
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15
+    ]
+  ],
+  "synchronous_singletons": true,
+  "async_work_handles_preserved": true
+}

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0038.json

@@ -0,0 +1,84 @@
+{
+  "rank": 38,
+  "world_diagnostics_gloo": true,
+  "aliases": [
+    [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15,
+      16,
+      17,
+      18,
+      19,
+      20,
+      21,
+      22,
+      23,
+      24,
+      25,
+      26,
+      27,
+      28,
+      29,
+      30,
+      31,
+      32,
+      33,
+      34,
+      35,
+      36,
+      37,
+      38,
+      39,
+      40,
+      41,
+      42,
+      43,
+      44,
+      45,
+      46,
+      47,
+      48,
+      49,
+      50,
+      51,
+      52,
+      53,
+      54,
+      55,
+      56,
+      57,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63
+    ],
+    [
+      32,
+      33,
+      34,
+      35,
+      36,
+      37,
+      38,
+      39
+    ]
+  ],
+  "synchronous_singletons": true,
+  "async_work_handles_preserved": true
+}

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0043.json

@@ -0,0 +1,84 @@
+{
+  "rank": 43,
+  "world_diagnostics_gloo": true,
+  "aliases": [
+    [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15,
+      16,
+      17,
+      18,
+      19,
+      20,
+      21,
+      22,
+      23,
+      24,
+      25,
+      26,
+      27,
+      28,
+      29,
+      30,
+      31,
+      32,
+      33,
+      34,
+      35,
+      36,
+      37,
+      38,
+      39,
+      40,
+      41,
+      42,
+      43,
+      44,
+      45,
+      46,
+      47,
+      48,
+      49,
+      50,
+      51,
+      52,
+      53,
+      54,
+      55,
+      56,
+      57,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63
+    ],
+    [
+      40,
+      41,
+      42,
+      43,
+      44,
+      45,
+      46,
+      47
+    ]
+  ],
+  "synchronous_singletons": true,
+  "async_work_handles_preserved": true
+}

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0051.json

@@ -0,0 +1,84 @@
+{
+  "rank": 51,
+  "world_diagnostics_gloo": true,
+  "aliases": [
+    [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15,
+      16,
+      17,
+      18,
+      19,
+      20,
+      21,
+      22,
+      23,
+      24,
+      25,
+      26,
+      27,
+      28,
+      29,
+      30,
+      31,
+      32,
+      33,
+      34,
+      35,
+      36,
+      37,
+      38,
+      39,
+      40,
+      41,
+      42,
+      43,
+      44,
+      45,
+      46,
+      47,
+      48,
+      49,
+      50,
+      51,
+      52,
+      53,
+      54,
+      55,
+      56,
+      57,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63
+    ],
+    [
+      48,
+      49,
+      50,
+      51,
+      52,
+      53,
+      54,
+      55
+    ]
+  ],
+  "synchronous_singletons": true,
+  "async_work_handles_preserved": true
+}

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0055.json

@@ -0,0 +1,84 @@
+{
+  "rank": 55,
+  "world_diagnostics_gloo": true,
+  "aliases": [
+    [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15,
+      16,
+      17,
+      18,
+      19,
+      20,
+      21,
+      22,
+      23,
+      24,
+      25,
+      26,
+      27,
+      28,
+      29,
+      30,
+      31,
+      32,
+      33,
+      34,
+      35,
+      36,
+      37,
+      38,
+      39,
+      40,
+      41,
+      42,
+      43,
+      44,
+      45,
+      46,
+      47,
+      48,
+      49,
+      50,
+      51,
+      52,
+      53,
+      54,
+      55,
+      56,
+      57,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63
+    ],
+    [
+      48,
+      49,
+      50,
+      51,
+      52,
+      53,
+      54,
+      55
+    ]
+  ],
+  "synchronous_singletons": true,
+  "async_work_handles_preserved": true
+}

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0061.json

@@ -0,0 +1,84 @@
+{
+  "rank": 61,
+  "world_diagnostics_gloo": true,
+  "aliases": [
+    [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15,
+      16,
+      17,
+      18,
+      19,
+      20,
+      21,
+      22,
+      23,
+      24,
+      25,
+      26,
+      27,
+      28,
+      29,
+      30,
+      31,
+      32,
+      33,
+      34,
+      35,
+      36,
+      37,
+      38,
+      39,
+      40,
+      41,
+      42,
+      43,
+      44,
+      45,
+      46,
+      47,
+      48,
+      49,
+      50,
+      51,
+      52,
+      53,
+      54,
+      55,
+      56,
+      57,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63
+    ],
+    [
+      56,
+      57,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63
+    ]
+  ],
+  "synchronous_singletons": true,
+  "async_work_handles_preserved": true
+}

f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0063.json

@@ -0,0 +1,84 @@
+{
+  "rank": 63,
+  "world_diagnostics_gloo": true,
+  "aliases": [
+    [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15,
+      16,
+      17,
+      18,
+      19,
+      20,
+      21,
+      22,
+      23,
+      24,
+      25,
+      26,
+      27,
+      28,
+      29,
+      30,
+      31,
+      32,
+      33,
+      34,
+      35,
+      36,
+      37,
+      38,
+      39,
+      40,
+      41,
+      42,
+      43,
+      44,
+      45,
+      46,
+      47,
+      48,
+      49,
+      50,
+      51,
+      52,
+      53,
+      54,
+      55,
+      56,
+      57,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63
+    ],
+    [
+      56,
+      57,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63
+    ]
+  ],
+  "synchronous_singletons": true,
+  "async_work_handles_preserved": true
+}